def _write_file_changelogs(entities, git_name, git_mail, agent): """Writes entity update/add changelogs, returns list of changelog paths Assembles appropriate changelog messages and then updates changelog for each entity. update_files() adds lists of updated and added File objects to entities in list. TODO should this go in DDR.changelog.py? @param entities: list of Entity objects. @param git_name: @param git_mail: @param agent: @returns: list of paths relative to repository base """ git_files = [] for entity in entities: messages = [] if getattr(entity, 'changelog_updated', None): for f in entity.changelog_updated: messages.append('Updated entity file {}'.format(f.json_path_rel)) #if getattr(entity, 'changelog_added', None): # for f in entity.changelog_added: # messages.append('Added entity file {}'.format(f.json_path_rel)) messages.append('@agent: %s' % agent) changelog.write_changelog_entry( entity.changelog_path, messages, user=git_name, email=git_mail) git_files.append(entity.changelog_path_rel) return git_files
def test_write_changelog_entry(tmpdir): path = str(tmpdir / 'changelog') user = '******' mail = '*****@*****.**' messages = ['testing', 'testing', '123'] timestamp = datetime(2014, 5, 29, 14, 38, 55, tzinfo=UTC) expected1 = '* testing\n* testing\n* 123\n' \ '-- gjost <*****@*****.**> Thu, 29 May 2014, 02:38 PM UTC \n' expected2 = '* testing\n* testing\n* 123\n' \ '-- gjost <*****@*****.**> Thu, 29 May 2014, 02:38 PM UTC \n\n' \ '* testing\n* testing\n* 123\n' \ '-- gjost <*****@*****.**> Thu, 29 May 2014, 02:38 PM UTC \n' # clean if os.path.exists(path): os.remove(path) # write once changelog.write_changelog_entry(path, messages, user, mail, timestamp) with open(path, 'r') as f1: first = f1.read() # write again changelog.write_changelog_entry(path, messages, user, mail, timestamp) with open(path, 'r') as f2: second = f2.read() # check assert first == expected1 assert second == expected2
def add_file_commit(entity, file_, repo, log, git_name, git_mail, agent): log.ok('add_file_commit(%s, %s, %s, %s, %s, %s)' % (file_, repo, log, git_name, git_mail, agent)) staged = dvcs.list_staged(repo) modified = dvcs.list_modified(repo) if staged and not modified: log.ok('All files staged.') log.ok('Updating changelog') path = file_.path_abs.replace('{}/'.format(entity.path), '') changelog_messages = ['Added entity file {}'.format(path)] if agent: changelog_messages.append('@agent: %s' % agent) changelog.write_changelog_entry( entity.changelog_path, changelog_messages, git_name, git_mail) log.ok('git add %s' % entity.changelog_path_rel) git_files = [entity.changelog_path_rel] dvcs.stage(repo, git_files) log.ok('Committing') commit = dvcs.commit(repo, 'Added entity file(s)', agent) log.ok('commit: {}'.format(commit.hexsha)) committed = dvcs.list_committed(repo, commit) committed.sort() log.ok('files committed:') for f in committed: log.ok('| %s' % f) else: log.not_ok('%s files staged, %s files modified' % (len(staged),len(modified))) log.not_ok('staged %s' % staged) log.not_ok('modified %s' % modified) log.not_ok('Can not commit!') raise Exception('Could not commit bc %s unstaged files: %s' % (len(modified), modified)) return file_,repo,log
def test_write_changelog_entry(): path = os.path.join(TESTING_BASE_DIR, 'changelog-%s' % datetime.now(TZ).strftime('%Y%m%d-%H%M%S')) user = '******' mail = '*****@*****.**' messages = ['testing', 'testing', '123'] timestamp = datetime(2014,5,29, 14,38,55,tzinfo=TZ) expected1 = '* testing\n* testing\n* 123\n' \ '-- gjost <*****@*****.**> Thu, 29 May 2014, 02:38 PM UTC \n' expected2 = '* testing\n* testing\n* 123\n' \ '-- gjost <*****@*****.**> Thu, 29 May 2014, 02:38 PM UTC \n\n' \ '* testing\n* testing\n* 123\n' \ '-- gjost <*****@*****.**> Thu, 29 May 2014, 02:38 PM UTC \n' # clean if os.path.exists(path): os.remove(path) # write once changelog.write_changelog_entry(path, messages, user, mail, timestamp) with open(path, 'r') as f1: first = f1.read() # write again changelog.write_changelog_entry(path, messages, user, mail, timestamp) with open(path, 'r') as f2: second = f2.read() # check assert first == expected1 assert second == expected2
def file_destroy(user_name, user_mail, collection, entity, rm_files, updated_files, agent='', commit=True): """Remove file and metadata - check that paths exist, etc - intantiate collection, repo objects - remove entity dir - update control and changelog - commit everything @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection: Collection @param entity: Entity @param rm_files: List of paths to files to delete (relative to entity files dir). @param updated_files: List of paths to updated file(s), relative to entitys. @param agent: (optional) Name of software making the change. @param commit: (optional) Commit files after staging them. @return: exit,message,touched_files ('ok' if successful) """ repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) # updated file paths are relative to collection root git_files = [f for f in updated_files] # remove the files # NOTE: File must be removed from filesystem at this point # so the File will be properly removed from the control file for f in rm_files: repo.git.rm('-rf', f) # update entity changelog changelog_files = [ # dont list access files in changelog # TODO use a models.File function to ID the original file f for f in rm_files if ('-a.jpg' not in f) and ('.json' not in f) ] changelog_messages = [ 'Deleted file {}'.format(os.path.basename(f)) for f in changelog_files ] if agent: changelog_messages.append('@agent: %s' % agent) write_changelog_entry(entity.changelog_path, changelog_messages, user_name, user_mail) git_files.append(entity.changelog_path_rel) dvcs.stage(repo, git_files) if commit: commit_obj = dvcs.commit(repo, 'Deleted file(s)', agent) return 0, 'ok', git_files
def file_destroy(user_name, user_mail, collection_path, entity_uid, rm_files, updated_files, agent=''): """Command-line function for creating an entity and adding it to the collection. - check that paths exist, etc - intantiate collection, repo objects - remove entity dir - update control and changelog - commit everything @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection_path: Absolute path to collection repo. @param entity_uid: A valid DDR entity UID @param rm_files: List of paths to files to delete (relative to entity files dir). @param updated_files: List of paths to updated file(s), relative to entitys. @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ collection = DDRCollection(collection_path) entity = DDREntity(collection.entity_path(entity_uid)) repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') if not GIT_REMOTE_NAME in [r.name for r in repo.remotes]: repo.create_remote(GIT_REMOTE_NAME, collection.git_url) # updated file paths are relative to collection root git_files = [os.path.join('files', entity.uid, f) for f in updated_files] # Only list the original file in changelog # TODO use a models.File function to ID the original file changelog_files = [f for f in rm_files if ('-a.jpg' not in f) and ('.json' not in f)] # remove the files # NOTE: entity files must be removed at this point so the entity will be # properly removed from the control file git = repo.git for f in rm_files: git.rm('-rf', f) # update entity control econtrol = entity.control() econtrol.update_checksums(entity) econtrol.write() git_files.append(econtrol.path_rel) # update entity changelog changelog_messages = ['Deleted entity file {}'.format(f) for f in changelog_files] if agent: changelog_messages.append('@agent: %s' % agent) write_changelog_entry(entity.changelog_path, changelog_messages, user_name, user_mail) git_files.append(entity.changelog_path_rel) # add files and commit commit_message = dvcs.compose_commit_message('Deleted entity file(s)', agent=agent) repo = commit_files(repo, commit_message, git_files, []) return 0,'ok'
def entity_destroy(user_name, user_mail, collection_path, entity_uid, agent=''): """Command-line function for creating an entity and adding it to the collection. - check that paths exist, etc - intantiate collection, repo objects - remove entity dir - update control and changelog - commit everything @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection_path: Absolute path to collection repo. @param entity_uid: A valid DDR entity UID @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ entity_dir = os.path.join(collection_path, 'files', entity_uid) if not os.path.exists(collection_path): raise Exception('collection_path not found: %s' % collection_path) if not os.path.exists(entity_dir): raise Exception('entity not found: %s' % entity_dir) collection = DDRCollection(collection_path) repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') if not GIT_REMOTE_NAME in [r.name for r in repo.remotes]: repo.create_remote(GIT_REMOTE_NAME, collection.git_url) git_files = [] # remove entity directory # NOTE: entity files must be removed at this point so the entity will be # properly removed from the control file git = repo.git git.rm('-rf', entity_dir) # update collection control ccontrol = collection.control() ccontrol.update_checksums(collection) ccontrol.write() git_files.append(ccontrol.path) # prep collection log entries changelog_messages = ['Deleted entity {}'.format(entity_uid),] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent) # collection changelog write_changelog_entry(collection.changelog_path, changelog_messages, user=user_name, email=user_mail) git_files.append(collection.changelog_path) # commit repo = commit_files(repo, commit_message, git_files) return 0,'ok'
def _write_entity_changelog(entity, git_name, git_mail, agent): msg = 'Updated entity file {}' messages = [ msg.format(entity.json_path), '@agent: %s' % agent, ] changelog.write_changelog_entry( entity.changelog_path, messages, user=git_name, email=git_mail)
def entity_destroy(user_name, user_mail, entity, updated_files, agent='', commit=True): """Command-line function for creating an entity and adding it to the collection. - check that paths exist, etc - intantiate collection, repo objects - remove entity dir - update control and changelog - commit everything @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param entity: Entity @param updated_files: List of paths to updated file(s), relative to entitys. @param agent: (optional) Name of software making the change. @param commit: (optional) Commit files after staging them. @return: message ('ok' if successful) """ collection = entity.collection() parent = entity.identifier.parent().object() repo = dvcs.repository(collection.path_abs, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) git_files = updated_files # remove entity directory # NOTE: entity files must be removed at this point so the entity will be # properly removed from the control file repo.git.rm('-rf', entity.path_abs) # prep collection log entries changelog_messages = [ 'Deleted entity {}'.format(entity.id), ] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent) # collection changelog write_changelog_entry(parent.changelog_path, changelog_messages, user=user_name, email=user_mail) git_files.append(parent.changelog_path) dvcs.stage(repo, git_files) # commit if commit: repo = commit_files(repo, commit_message, git_files) return 0, 'ok'
def entity_destroy(user_name, user_mail, collection, entity, agent=''): """Command-line function for creating an entity and adding it to the collection. - check that paths exist, etc - intantiate collection, repo objects - remove entity dir - update control and changelog - commit everything @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection: Collection @param entity: Entity @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ if not os.path.exists(collection.path_abs): raise Exception('collection_path not found: %s' % collection.path_abs) if not os.path.exists(entity.path_abs): raise Exception('entity not found: %s' % entity.path_abs) repo = dvcs.repository(collection.path_abs, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) git_files = [] # remove entity directory # NOTE: entity files must be removed at this point so the entity will be # properly removed from the control file git = repo.git git.rm('-rf', entity.path_abs) # update collection control ccontrol = collection.control() ccontrol.update_checksums(collection) ccontrol.write() git_files.append(ccontrol.path) # prep collection log entries changelog_messages = ['Deleted entity {}'.format(entity.id),] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent) # collection changelog write_changelog_entry(collection.changelog_path, changelog_messages, user=user_name, email=user_mail) git_files.append(collection.changelog_path) # commit repo = commit_files(repo, commit_message, git_files) return 0,'ok'
def entity_update(user_name, user_mail, collection, entity, updated_files, agent='', commit=True): """Command-line function for committing changes to the specified entity file. NOTE: Does not push to the workbench server. Updates entity changelog but NOT in collection changelog. Makes an entry in git log. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection: Collection @param entity: Entity @param updated_files: List of paths to updated file(s), relative to entitys. @param agent: (optional) Name of software making the change. @param commit: (optional) Commit files after staging them. @return: message ('ok' if successful) """ repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) # entity file paths are relative to collection root git_files = [] for f in updated_files: git_files.append(os.path.join('files', entity.id, str(f))) # entity changelog entity_changelog_messages = [] for f in updated_files: p = os.path.join(entity.id, f) entity_changelog_messages.append('Updated entity file {}'.format(p)) # prep log entries if agent: entity_changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message('Updated entity file(s)', agent=agent) write_changelog_entry(entity.changelog_path, entity_changelog_messages, user=user_name, email=user_mail) git_files.append(entity.changelog_path_rel) if commit: # add files and commit repo = commit_files(repo, commit_message, git_files, []) return 0, 'ok'
def entity_update(user_name, user_mail, collection_path, entity_uid, updated_files, agent=''): """Command-line function for committing changes to the specified entity file. NOTE: Does not push to the workbench server. Updates entity changelog but NOT in collection changelog. Makes an entry in git log. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection_path: Absolute path to collection repo. @param entity_uid: A valid DDR entity UID @param updated_files: List of paths to updated file(s), relative to entitys. @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ collection = DDRCollection(collection_path) entity = DDREntity(collection.entity_path(entity_uid)) repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') if not GIT_REMOTE_NAME in [r.name for r in repo.remotes]: repo.create_remote(GIT_REMOTE_NAME, collection.git_url) # entity file paths are relative to collection root git_files = [] for f in updated_files: git_files.append( os.path.join( 'files', entity.uid, f) ) # entity changelog entity_changelog_messages = [] for f in updated_files: p = os.path.join(entity.uid, f) entity_changelog_messages.append('Updated entity file {}'.format(p)) # prep log entries if agent: entity_changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message('Updated entity file(s)', agent=agent) write_changelog_entry(entity.changelog_path, entity_changelog_messages, user=user_name, email=user_mail) git_files.append(entity.changelog_path_rel) # add files and commit repo = commit_files(repo, commit_message, git_files, []) return 0,'ok'
def entity_destroy(user_name, user_mail, entity, updated_files, agent='', commit=True): """Command-line function for creating an entity and adding it to the collection. - check that paths exist, etc - intantiate collection, repo objects - remove entity dir - update control and changelog - commit everything @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param entity: Entity @param updated_files: List of paths to updated file(s), relative to entitys. @param agent: (optional) Name of software making the change. @param commit: (optional) Commit files after staging them. @return: message ('ok' if successful) """ collection = entity.collection() parent = entity.identifier.parent().object() repo = dvcs.repository(collection.path_abs, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) git_files = updated_files # remove entity directory # NOTE: entity files must be removed at this point so the entity will be # properly removed from the control file repo.git.rm('-rf', entity.path_abs) # prep collection log entries changelog_messages = ['Deleted entity {}'.format(entity.id),] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent) # collection changelog write_changelog_entry(parent.changelog_path, changelog_messages, user=user_name, email=user_mail) git_files.append(parent.changelog_path) dvcs.stage(repo, git_files) # commit if commit: repo = commit_files(repo, commit_message, git_files) return 0,'ok'
def entity_update(user_name, user_mail, collection, entity, updated_files, agent='', commit=True): """Command-line function for committing changes to the specified entity file. NOTE: Does not push to the workbench server. Updates entity changelog but NOT in collection changelog. Makes an entry in git log. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection: Collection @param entity: Entity @param updated_files: List of paths to updated file(s), relative to entitys. @param agent: (optional) Name of software making the change. @param commit: (optional) Commit files after staging them. @return: message ('ok' if successful) """ repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) # entity file paths are relative to collection root git_files = [] for f in updated_files: git_files.append( os.path.join( 'files', entity.id, str(f)) ) # entity changelog entity_changelog_messages = [] for f in updated_files: p = os.path.join(entity.id, f) entity_changelog_messages.append('Updated entity file {}'.format(p)) # prep log entries if agent: entity_changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message('Updated entity file(s)', agent=agent) write_changelog_entry(entity.changelog_path, entity_changelog_messages, user=user_name, email=user_mail) git_files.append(entity.changelog_path_rel) if commit: # add files and commit repo = commit_files(repo, commit_message, git_files, []) return 0,'ok'
def update(user_name, user_mail, collection, updated_files, agent='', commit=False): """Command-line function for commiting changes to the specified file. NOTE: Does not push to the workbench server. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection: Collection @param updated_files: List of relative paths to updated file(s). @param agent: (optional) Name of software making the change. @param commit: (optional) Commit files after staging them. @return: message ('ok' if successful) """ repo = dvcs.repository(collection.path, user_name, user_mail) if repo: logging.debug(' git repo {}'.format(collection.path)) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) # prep log entries changelog_messages = [] for f in updated_files: changelog_messages.append('Updated collection file(s) {}'.format(f)) if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message('Updated metadata file(s)', agent=agent) # write changelog write_changelog_entry(collection.changelog_path, changelog_messages, user_name, user_mail) if os.path.exists(collection.changelog_path): updated_files.append(collection.changelog_path) else: logging.error(' COULD NOT UPDATE changelog') if commit: # add files and commit repo = commit_files(repo, commit_message, updated_files, []) return 0, 'ok'
def update(user_name, user_mail, collection_path, updated_files, agent=''): """Command-line function for commiting changes to the specified file. NOTE: Does not push to the workbench server. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection_path: Absolute path to collection repo. @param updated_files: List of relative paths to updated file(s). @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ collection = DDRCollection(collection_path) repo = dvcs.repository(collection.path, user_name, user_mail) if repo: logging.debug(' git repo {}'.format(collection.path)) repo.git.checkout('master') if not GIT_REMOTE_NAME in [r.name for r in repo.remotes]: repo.create_remote(GIT_REMOTE_NAME, collection.git_url) # prep log entries changelog_messages = [] for f in updated_files: changelog_messages.append('Updated collection file(s) {}'.format(f)) if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message('Updated metadata file(s)', agent=agent) # write changelog write_changelog_entry(collection.changelog_path, changelog_messages, user_name, user_mail) if os.path.exists(collection.changelog_path): updated_files.append(collection.changelog_path) else: logging.error(' COULD NOT UPDATE changelog') # add files and commit repo = commit_files(repo, commit_message, updated_files, []) return 0,'ok'
def entity_annex_add(user_name, user_mail, collection_path, entity_uid, updated_files, new_annex_files, agent='', entity=None): """Command-line function for git annex add-ing a file and updating metadata. All this function does is git annex add the file, update changelog and mets.xml, and commit. It does not copy the file into the entity dir. It does not mark the file as master/mezzanine/access/etc or edit any metadata. It does not perform any background processing on the file. TODO Refactor this when ddr-local models moved into ddr-cmdln WARNING - UGLY HACK! The 'entity' arg is intended to allow ddr-local to pass in Entity objects and use their checksums() method. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection_path: Absolute path to collection repo. @param entity_uid: A valid DDR entity UID @param updated_files: list of paths to updated files (relative to collection repo). @param new_annex_files: List of paths to new files (relative to entity files dir). @param agent: (optional) Name of software making the change. @param entity: (optional) Entity object (see above) @return: message ('ok' if successful) """ collection = DDRCollection(collection_path) if not entity: entity = DDREntity(collection.entity_path(entity_uid)) repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') if not GIT_REMOTE_NAME in [r.name for r in repo.remotes]: repo.create_remote(GIT_REMOTE_NAME, collection.git_url) git_files = [] annex_files = [] if not os.path.exists(collection.annex_path): logging.error(' .git/annex IS MISSING!') return 1,'.git/annex IS MISSING!' if not os.path.exists(entity.path): logging.error(' Entity does not exist: {}'.format(entity.uid)) return 1,'entity does not exist: {}'.format(entity.uid) if not os.path.exists(entity.files_path): logging.error(' Entity files_path does not exist: {}'.format(entity.uid)) return 1,'entity files_path does not exist: {}'.format(entity.uid) # new annex files new_files_rel_entity = [] for new_file in new_annex_files: # paths: absolute, relative to collection repo, relative to entity_dir new_file_abs = os.path.join(entity.files_path, new_file) if not os.path.exists(new_file_abs): logging.error(' File does not exist: {}'.format(new_file_abs)) return 1,'File does not exist: {}'.format(new_file_abs) new_file_rel = os.path.join(entity.files_path_rel, new_file) new_file_rel_entity = new_file_abs.replace('{}/'.format(entity.path), '') new_files_rel_entity.append(new_file_rel_entity) annex_files.append(new_file_rel) # updated files [git_files.append(updated_file) for updated_file in updated_files] # update entity control econtrol = entity.control() econtrol.update_checksums(entity) econtrol.write() git_files.append(econtrol.path_rel) # prep log entries changelog_messages = ['Added entity file {}'.format(f) for f in new_files_rel_entity] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message('Added entity file(s)', agent=agent) # update entity changelog write_changelog_entry(entity.changelog_path, changelog_messages, user_name, user_mail) git_files.append(entity.changelog_path_rel) # add files and commit repo = commit_files(repo, commit_message, git_files, annex_files) return 0,'ok'
def create(user_name, user_mail, identifier, templates, agent=''): """Command-line function for creating a new collection. Clones a blank collection object from workbench server, adds files, commits. - clones new repo from gitolite server # Easier to have Gitolite create repo then clone (http://sitaramc.github.com/gitolite/repos.html) # than to add existing to Gitolite (http://sitaramc.github.com/gitolite/rare.html#existing). local requests CID from workbench API background:collection init: $ collection -cCID -oinit] background:collection init: $ git clone git@mits:ddr-ORG-C $ git clone git@mits:ddr-densho-1 Cloning into 'ddr-densho-1'... Initialized empty Git repository in /home/git/repositories/ddr-densho-1.git/ warning: You appear to have cloned an empty repository. background:entity init: $ git annex init background:entity init: $ git add changelog control ead.xml .gitignore background:entity init: $ git commit @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param identifier: Identifier @param templates: List of metadata templates (absolute paths). @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ gitolite = dvcs.Gitolite(config.GITOLITE) gitolite.initialize() if identifier.id in gitolite.collections(): raise Exception("'%s' already exists -- clone instead." % identifier.id) git_url = '{}:{}.git'.format(config.GITOLITE, identifier.id) repo = git.Repo.clone_from(git_url, identifier.path_abs()) logging.debug(' git clone {}'.format(git_url)) if repo: logging.debug(' OK') else: logging.error(' COULD NOT CLONE!') if os.path.exists(identifier.path_abs('git')): logging.debug(' .git/ is present') else: logging.error(' .git/ IS MISSING!') # there is no master branch at this point dvcs.remote_add(repo, git_url, config.GIT_REMOTE_NAME) dvcs.git_set_configs(repo, user_name, user_mail) dvcs.annex_set_configs(repo, user_name, user_mail) git_files = [] # copy template files to collection for src in templates: if os.path.exists(src): dst = os.path.join(identifier.path_abs(), os.path.basename(src)) logging.debug('cp %s, %s' % (src, dst)) shutil.copy(src, dst) if os.path.exists(dst): git_files.append(dst) else: logging.error('COULD NOT COPY %s' % src) # instantiate now that we have collection dir and some templates object_class = identifier.object_class() collection = object_class(identifier.path_abs()) # add control, .gitignore, changelog control = collection.control() gitignore = collection.gitignore() # prep log entries changelog_messages = ['Initialized collection {}'.format(collection.id)] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent) write_changelog_entry(collection.changelog_path, changelog_messages, user_name, user_mail) if os.path.exists(control.path): git_files.append(control.path_rel) else: logging.error(' COULD NOT CREATE control') if os.path.exists(collection.gitignore_path): git_files.append(collection.gitignore_path_rel) else: logging.error(' COULD NOT CREATE .gitignore') if os.path.exists(collection.changelog_path): git_files.append(collection.changelog_path_rel) else: logging.error(' COULD NOT CREATE changelog') # add files and commit repo = commit_files(repo, commit_message, git_files, []) # master branch should be created by this point # git annex init logging.debug(' git annex init') repo.git.annex('init') if os.path.exists(os.path.join(collection.path, '.git', 'annex')): logging.debug(' .git/annex/ OK') else: logging.error(' .git/annex/ IS MISSING!') # manual version of git-annex-sync - see notes for DDR.commands.sync. logging.debug('git push %s git-annex' % config.GIT_REMOTE_NAME) repo.git.checkout('git-annex') repo.git.push(config.GIT_REMOTE_NAME, 'git-annex') logging.debug('git push %s master' % config.GIT_REMOTE_NAME) repo.git.checkout('master') repo.git.push(config.GIT_REMOTE_NAME, 'master') logging.debug('OK') drive_label = storage.drive_label(repo.working_dir) dvcs.annex_set_description(repo, dvcs.annex_status(repo), drive_label=drive_label) return 0,'ok'
def entity_create(user_name, user_mail, collection, eidentifier, updated_files, templates, agent=''): """Command-line function for creating an entity and adding it to the collection. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection: Collection @param eidentifier: Identifier @param updated_files: List of updated files (relative to collection root). @param templates: List of entity metadata templates (absolute paths). @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) git_files = [] # entity dir if not os.path.exists(eidentifier.path_abs()): os.makedirs(eidentifier.path_abs()) # copy template files to entity for src in templates: if os.path.exists(src): dst = os.path.join(eidentifier.path_abs(), os.path.basename(src)) logging.debug('cp %s, %s' % (src, dst)) shutil.copy(src, dst) if os.path.exists(dst): git_files.append(dst) else: logging.error('COULD NOT COPY %s' % src) # instantiate now that we have entity dir and some templates object_class = eidentifier.object_class() entity = object_class(eidentifier.path_abs()) # entity control econtrol = entity.control() if os.path.exists(econtrol.path): git_files.append(econtrol.path) else: logging.error(' COULD NOT CREATE control') # update collection control ccontrol = collection.control() ccontrol.update_checksums(collection) ccontrol.write() git_files.append(ccontrol.path) # prep ENTITY log entries entity_changelog_messages = ['Initialized entity {}'.format(entity.id),] if agent: entity_changelog_messages.append('@agent: %s' % agent) # prep COLLECTION log entries changelog_messages = ['Initialized entity {}'.format(entity.id),] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent) # ENTITY changelog write_changelog_entry(entity.changelog_path, entity_changelog_messages, user=user_name, email=user_mail) if os.path.exists(entity.changelog_path): git_files.append(entity.changelog_path) else: logging.error(' COULD NOT CREATE changelog') # COLLECTION changelog write_changelog_entry(collection.changelog_path, changelog_messages, user=user_name, email=user_mail) git_files.append(collection.changelog_path) # add updated collection files for src in updated_files: git_files.append(src) # add files and commit repo = commit_files(repo, commit_message, git_files, []) return 0,'ok'
def entity_annex_add(user_name, user_mail, collection, entity, updated_files, new_annex_files, agent=''): """Command-line function for git annex add-ing a file and updating metadata. All this function does is git annex add the file, update changelog and mets.xml, and commit. It does not copy the file into the entity dir. It does not mark the file as master/mezzanine/access/etc or edit any metadata. It does not perform any background processing on the file. TODO Refactor this when ddr-local models moved into ddr-cmdln WARNING - UGLY HACK! The 'entity' arg is intended to allow ddr-local to pass in Entity objects and use their checksums() method. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection: Collection @param entity: Entity @param updated_files: list of paths to updated files (relative to collection repo). @param new_annex_files: List of paths to new files (relative to entity files dir). @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) git_files = [] annex_files = [] if not os.path.exists(collection.annex_path): logging.error(' .git/annex IS MISSING!') return 1,'.git/annex IS MISSING!' if not os.path.exists(entity.path): logging.error(' Entity does not exist: {}'.format(entity.id)) return 1,'entity does not exist: {}'.format(entity.id) if not os.path.exists(entity.files_path): logging.error(' Entity files_path does not exist: {}'.format(entity.id)) return 1,'entity files_path does not exist: {}'.format(entity.id) # new annex files new_files_rel_entity = [] for new_file in new_annex_files: # paths: absolute, relative to collection repo, relative to entity_dir new_file_abs = os.path.join(entity.files_path, new_file) if not os.path.exists(new_file_abs): logging.error(' File does not exist: {}'.format(new_file_abs)) return 1,'File does not exist: {}'.format(new_file_abs) new_file_rel = os.path.join(entity.files_path_rel, new_file) new_file_rel_entity = new_file_abs.replace('{}/'.format(entity.path), '') new_files_rel_entity.append(new_file_rel_entity) annex_files.append(new_file_rel) # updated files [git_files.append(updated_file) for updated_file in updated_files] # update entity control econtrol = entity.control() econtrol.update_checksums(entity) econtrol.write() git_files.append(econtrol.path_rel) # prep log entries changelog_messages = ['Added entity file {}'.format(f) for f in new_files_rel_entity] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message('Added entity file(s)', agent=agent) # update entity changelog write_changelog_entry(entity.changelog_path, changelog_messages, user_name, user_mail) git_files.append(entity.changelog_path_rel) # add files and commit repo = commit_files(repo, commit_message, git_files, annex_files) return 0,'ok'
def entity_create(user_name, user_mail, collection_path, entity_uid, updated_files, templates, agent=''): """Command-line function for creating an entity and adding it to the collection. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection_path: Absolute path to collection repo. @param entity_uid: A valid DDR entity UID @param updated_files: List of updated files (relative to collection root). @param templates: List of entity metadata templates (absolute paths). @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ collection = DDRCollection(collection_path) entity = DDREntity(collection.entity_path(entity_uid)) repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') if not GIT_REMOTE_NAME in [r.name for r in repo.remotes]: repo.create_remote(GIT_REMOTE_NAME, collection.git_url) git_files = [] # entity dir if not os.path.exists(entity.path): os.makedirs(entity.path) # copy template files to entity for src in templates: if os.path.exists(src): dst = os.path.join(entity.path, os.path.basename(src)) logging.debug('cp %s, %s' % (src, dst)) shutil.copy(src, dst) if os.path.exists(dst): git_files.append(dst) else: logging.error('COULD NOT COPY %s' % src) # entity control econtrol = entity.control() if os.path.exists(econtrol.path): git_files.append(econtrol.path) else: logging.error(' COULD NOT CREATE control') # update collection control ccontrol = collection.control() ccontrol.update_checksums(collection) ccontrol.write() git_files.append(ccontrol.path) # prep ENTITY log entries entity_changelog_messages = ['Initialized entity {}'.format(entity.uid),] if agent: entity_changelog_messages.append('@agent: %s' % agent) # prep COLLECTION log entries changelog_messages = ['Initialized entity {}'.format(entity.uid),] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent) # ENTITY changelog write_changelog_entry(entity.changelog_path, entity_changelog_messages, user=user_name, email=user_mail) if os.path.exists(entity.changelog_path): git_files.append(entity.changelog_path) else: logging.error(' COULD NOT CREATE changelog') # COLLECTION changelog write_changelog_entry(collection.changelog_path, changelog_messages, user=user_name, email=user_mail) git_files.append(collection.changelog_path) # add updated collection files for src in updated_files: git_files.append(src) # add files and commit repo = commit_files(repo, commit_message, git_files, []) return 0,'ok'
def create(user_name, user_mail, collection_path, templates, agent=''): """Command-line function for creating a new collection. Clones a blank collection object from workbench server, adds files, commits. - clones new repo from gitolite server # Easier to have Gitolite create repo then clone (http://sitaramc.github.com/gitolite/repos.html) # than to add existing to Gitolite (http://sitaramc.github.com/gitolite/rare.html#existing). local requests CID from workbench API background:collection init: $ collection -cCID -oinit] background:collection init: $ git clone git@mits:ddr-ORG-C $ git clone git@mits:ddr-densho-1 Cloning into 'ddr-densho-1'... Initialized empty Git repository in /home/git/repositories/ddr-densho-1.git/ warning: You appear to have cloned an empty repository. background:entity init: $ git annex init background:entity init: $ git add changelog control ead.xml .gitignore background:entity init: $ git commit @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection_path: Absolute path to collection repo. @param templates: List of metadata templates (absolute paths). @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ collection = DDRCollection(collection_path) url = '{}:{}.git'.format(GITOLITE, collection.uid) repo = git.Repo.clone_from(url, collection.path) logging.debug(' git clone {}'.format(url)) if repo: logging.debug(' OK') else: logging.error(' COULD NOT CLONE!') if os.path.exists(os.path.join(collection.path, '.git')): logging.debug(' .git/ is present') else: logging.error(' .git/ IS MISSING!') # there is no master branch at this point if not GIT_REMOTE_NAME in [r.name for r in repo.remotes]: repo.create_remote(GIT_REMOTE_NAME, collection.git_url) repo = dvcs.set_git_configs(repo, user_name, user_mail) git_files = [] # copy template files to collection for src in templates: if os.path.exists(src): dst = os.path.join(collection.path, os.path.basename(src)) logging.debug('cp %s, %s' % (src, dst)) shutil.copy(src, dst) if os.path.exists(dst): git_files.append(dst) else: logging.error('COULD NOT COPY %s' % src) # add control, .gitignore, changelog control = collection.control() gitignore = collection.gitignore() # prep log entries changelog_messages = ['Initialized collection {}'.format(collection.uid)] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent) write_changelog_entry(collection.changelog_path, changelog_messages, user_name, user_mail) if os.path.exists(control.path): git_files.append(control.path_rel) else: logging.error(' COULD NOT CREATE control') if os.path.exists(collection.gitignore_path): git_files.append(collection.gitignore_path_rel) else: logging.error(' COULD NOT CREATE .gitignore') if os.path.exists(collection.changelog_path): git_files.append(collection.changelog_path_rel) else: logging.error(' COULD NOT CREATE changelog') # add files and commit repo = commit_files(repo, commit_message, git_files, []) # master branch should be created by this point # git annex init logging.debug(' git annex init') repo.git.annex('init') if os.path.exists(os.path.join(collection.path, '.git', 'annex')): logging.debug(' .git/annex/ OK') else: logging.error(' .git/annex/ IS MISSING!') # manual version of git-annex-sync - see notes for DDR.commands.sync. logging.debug('git push %s git-annex' % GIT_REMOTE_NAME) repo.git.checkout('git-annex') repo.git.push(GIT_REMOTE_NAME, 'git-annex') logging.debug('git push %s master' % GIT_REMOTE_NAME) repo.git.checkout('master') repo.git.push(GIT_REMOTE_NAME, 'master') logging.debug('OK') dvcs.set_annex_description(repo) return 0,'ok'
def entity_create(user_name, user_mail, collection, eidentifier, updated_files, agent=''): """Command-line function for creating an entity and adding it to the collection. @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection: Collection @param eidentifier: Identifier @param updated_files: List of updated files (relative to collection root). @param agent: (optional) Name of software making the change. @return: message ('ok' if successful) """ repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) git_files = [] # entity dir if not os.path.exists(eidentifier.path_abs()): os.makedirs(eidentifier.path_abs()) # instantiate and write JSON,XML object_class = eidentifier.object_class() entity = object_class.new(eidentifier) entity.write_json() entity.write_xml() git_files.append(eidentifier.path_rel('json')) git_files.append(eidentifier.path_rel('xml')) # entity control econtrol = entity.control() if os.path.exists(econtrol.path): git_files.append(econtrol.path) else: logging.error(' COULD NOT CREATE control') # update collection control ccontrol = collection.control() ccontrol.update_checksums(collection) ccontrol.write() git_files.append(ccontrol.path) # prep ENTITY log entries entity_changelog_messages = [ 'Initialized entity {}'.format(entity.id), ] if agent: entity_changelog_messages.append('@agent: %s' % agent) # prep COLLECTION log entries changelog_messages = [ 'Initialized entity {}'.format(entity.id), ] if agent: changelog_messages.append('@agent: %s' % agent) commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent) # ENTITY changelog write_changelog_entry(entity.changelog_path, entity_changelog_messages, user=user_name, email=user_mail) if os.path.exists(entity.changelog_path): git_files.append(entity.changelog_path) else: logging.error(' COULD NOT CREATE changelog') # COLLECTION changelog write_changelog_entry(collection.changelog_path, changelog_messages, user=user_name, email=user_mail) git_files.append(collection.changelog_path) # add updated collection files for src in updated_files: git_files.append(src) # add files and commit repo = commit_files(repo, commit_message, git_files, []) return 0, 'ok'
def file_destroy(user_name, user_mail, collection, entity, rm_files, updated_files, agent='', commit=True): """Remove file and metadata - check that paths exist, etc - intantiate collection, repo objects - remove entity dir - update control and changelog - commit everything @param user_name: Username for use in changelog, git log @param user_mail: User email address for use in changelog, git log @param collection: Collection @param entity: Entity @param rm_files: List of paths to files to delete (relative to entity files dir). @param updated_files: List of paths to updated file(s), relative to entitys. @param agent: (optional) Name of software making the change. @param commit: (optional) Commit files after staging them. @return: exit,message,touched_files ('ok' if successful) """ repo = dvcs.repository(collection.path, user_name, user_mail) repo.git.checkout('master') dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME) # updated file paths are relative to collection root git_files = [os.path.join('files', entity.id, f) for f in updated_files] # remove the files # NOTE: File must be removed from filesystem at this point # so the File will be properly removed from the control file for f in rm_files: repo.git.rm('-rf', f) # update entity control econtrol = entity.control() econtrol.update_checksums(entity) econtrol.write() git_files.append(econtrol.path_rel) # update entity changelog changelog_files = [ # dont list access files in changelog # TODO use a models.File function to ID the original file f for f in rm_files if ('-a.jpg' not in f) and ('.json' not in f) ] changelog_messages = [ 'Deleted file {}'.format(os.path.basename(f)) for f in changelog_files ] if agent: changelog_messages.append('@agent: %s' % agent) write_changelog_entry( entity.changelog_path, changelog_messages, user_name, user_mail ) git_files.append(entity.changelog_path_rel) dvcs.stage(repo, git_files) if commit: commit_obj = dvcs.commit(repo, 'Deleted file(s)', agent) return 0,'ok',git_files