def save(collection_path, cleaned_data, git_name, git_mail):
"""The time-consuming parts of collection-edit.
@param collection_path: str Absolute path to collection
@param cleaned_data: dict form.cleaned_data
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
"""
logger.debug('tasks.collection.save(%s,%s,%s)' % (
git_name, git_mail, collection_path))
collection = Collection.from_identifier(Identifier(path=collection_path))
gitstatus.lock(settings.MEDIA_BASE, 'collection_edit')
exit,status,updated_files = collection.save(
git_name, git_mail,
cleaned_data
)
dvcs_tasks.gitstatus_update.apply_async(
(collection_path,),
countdown=2
)
return status,collection_path
def reindex( index ):
"""
@param index: Name of index to create or update
"""
gitstatus.lock(settings.MEDIA_BASE, 'reindex')
logger.debug('------------------------------------------------------------------------')
logger.debug('webui.tasks.reindex(%s)' % index)
statuses = []
if not os.path.exists(settings.MEDIA_BASE):
raise NameError('MEDIA_BASE does not exist - you need to remount!')
logger.debug('webui.tasks.reindex(%s)' % index)
logger.debug('DOCSTORE_HOSTS: %s' % settings.DOCSTORE_HOSTS)
logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
logger.debug('deleting existing index: %s' % index)
delete_status = docstore.delete_index(settings.DOCSTORE_HOSTS, index)
logger.debug(delete_status)
logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
logger.debug('creating new index: %s' % index)
create_status = docstore.create_index(settings.DOCSTORE_HOSTS, index)
logger.debug(create_status)
logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
logger.debug('mappings: %s, %s' % (docstore.HARD_CODED_MAPPINGS_PATH, models.MODELS_DIR))
mappings_status = docstore.put_mappings(settings.DOCSTORE_HOSTS, index,
docstore.HARD_CODED_MAPPINGS_PATH, models.MODELS_DIR)
logger.debug(mappings_status)
logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
logger.debug('facets')
facets_status = docstore.put_facets(settings.DOCSTORE_HOSTS, index)
logger.debug(facets_status)
logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
logger.debug('indexing')
index_status = docstore.index(settings.DOCSTORE_HOSTS, index, path=settings.MEDIA_BASE,
recursive=True, public=False)
logger.debug(index_status)
return statuses
def signatures(collection_path, git_name, git_mail):
"""Identifies signature files for collection and entities.
@param collection_path: Absolute path to collection repo.
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@return collection_path: Absolute path to collection.
"""
gitstatus.lock(settings.MEDIA_BASE, 'collection_signatures')
collection = Collection.from_identifier(Identifier(path=collection_path))
updates = signatures.find_updates(collection)
files_written = signatures.write_updates(updates)
# TODO move this code to webui.models.Collection
status,msg = signatures.commit_updates(
collection,
files_written,
git_name, git_mail, agent='ddr-local'
)
logger.debug('DONE')
logger.debug('Updating Elasticsearch')
if settings.DOCSTORE_ENABLED:
collection = Collection.from_identifier(Identifier(path=collection_path))
try:
collection.post_json()
except ConnectionError:
logger.error('Could not update search index')
return collection_path
def sync( git_name, git_mail, collection_path ):
"""Synchronizes collection repo with workbench server.
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param collection_path: Absolute path to collection repo.
@return collection_path: Absolute path to collection.
"""
gitstatus.lock(settings.MEDIA_BASE, 'collection_sync')
ci = Identifier(path=collection_path)
collection = Collection.from_identifier(ci)
# TODO move this code to webui.models.Collection.sync
exit,status = commands.sync(
git_name, git_mail,
collection
)
logger.debug('Updating Elasticsearch')
if settings.DOCSTORE_ENABLED:
try:
collection.reindex()
except ConnectionError:
logger.error('Could not update search index')
return collection_path
def entity_edit(collection_path, entity_id, form_data, git_name, git_mail, agent=''):
"""The time-consuming parts of entity-edit.
@param collection_path: str Absolute path to collection
@param entity_id: str
@param form_data: dict
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
logger.debug('tasks.entity.entity_edit(%s,%s,%s,%s,%s)' % (
git_name, git_mail, collection_path, entity_id, agent))
collection = Collection.from_identifier(Identifier(path=collection_path))
entity = Entity.from_identifier(Identifier(id=entity_id))
gitstatus.lock(settings.MEDIA_BASE, 'entity_edit')
exit,status,updated_files = entity.save(
git_name, git_mail,
collection,
form_data
)
dvcs_tasks.gitstatus_update.apply_async(
(collection.path,),
countdown=2
)
return status,collection_path,entity_id
def delete_file( git_name, git_mail, collection_path, entity_id, file_basename, agent='' ):
"""
@param collection_path: string
@param entity_id: string
@param file_basename: string
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
logger.debug('delete_file(%s,%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, file_basename, agent))
gitstatus.lock(settings.MEDIA_BASE, 'delete_file')
file_id = os.path.splitext(file_basename)[0]
file_ = DDRFile.from_identifier(Identifier(file_id))
entity = Entity.from_identifier(Identifier(entity_id))
collection = Collection.from_identifier(Identifier(path=collection_path))
logger.debug('delete from repository')
rm_files,updated_files = entity.prep_rm_file(file_)
status,message = commands.file_destroy(
git_name, git_mail,
collection, entity,
rm_files, updated_files,
agent
)
logger.debug('delete from search index')
try:
docstore.delete(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, file_.id)
except ConnectionError:
logger.error('Could not delete document from Elasticsearch.')
return status,message,collection_path,file_basename
def delete_file( git_name, git_mail, collection_path, entity_id, file_basename, agent='' ):
"""
@param collection_path: string
@param entity_id: string
@param file_basename: string
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
logger.debug('delete_file(%s,%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, file_basename, agent))
gitstatus.lock(settings.MEDIA_BASE, 'delete_file')
# TODO rm_files list should come from the File model
file_id = os.path.splitext(file_basename)[0]
repo,org,cid,eid,role,sha1 = file_id.split('-')
entity = Entity.from_json(Entity.entity_path(None,repo,org,cid,eid))
file_ = entity.file(repo, org, cid, eid, role, sha1)
rm_files = file_.files_rel(collection_path)
logger.debug('rm_files: %s' % rm_files)
# remove file from entity.json
# TODO move this to commands.file_destroy or models.Entity
for f in entity.files:
if f.basename == file_basename:
entity.files.remove(f)
entity.dump_json()
updated_files = ['entity.json']
logger.debug('updated_files: %s' % updated_files)
status,message = file_destroy(git_name, git_mail, collection_path, entity_id, rm_files, updated_files, agent)
return status,message,collection_path,file_basename
def entity_add_access( git_name, git_mail, entity, ddrfile, agent='' ):
"""
@param entity: Entity
@param ddrfile: DDRFile
@param src_path: Absolute path to an uploadable file.
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
gitstatus.lock(settings.MEDIA_BASE, 'entity_add_access')
return entity.add_access(git_name, git_mail, ddrfile, agent)
def delete_entity( git_name, git_mail, collection_path, entity_id, agent='' ):
"""
@param collection_path: string
@param entity_id: string
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
gitstatus.lock(settings.MEDIA_BASE, 'delete_entity')
logger.debug('collection_delete_entity(%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, agent))
status,message = entity_destroy(git_name, git_mail, collection_path, entity_id, agent)
return status,message,collection_path,entity_id
def entity_add_file( git_name, git_mail, entity, src_path, role, data, agent='' ):
"""
@param entity: Entity
@param src_path: Absolute path to an uploadable file.
@param role: Keyword of a file role.
@param data: Dict containing form data.
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
gitstatus.lock(settings.MEDIA_BASE, 'entity_add_file')
return entity.add_file(git_name, git_mail, src_path, role, data, agent)
def test_locked_global(self):
os.makedirs(os.path.join(BASEDIR, 'tmp'))
lock_path = os.path.join(BASEDIR, 'tmp', 'gitstatus-lock')
self.assertEqual(gitstatus.locked_global(BASEDIR), False)
# lock
tids = ['1234', '1248']
lockfile0 = gitstatus.lock(BASEDIR, tids[0])
self.assertNotEqual(gitstatus.locked_global(BASEDIR), False)
lockfile1 = gitstatus.lock(BASEDIR, tids[1])
self.assertNotEqual(gitstatus.locked_global(BASEDIR), False)
# unlock
unlocked0 = gitstatus.unlock(BASEDIR, tids[0])
self.assertNotEqual(gitstatus.locked_global(BASEDIR), False)
unlocked1 = gitstatus.unlock(BASEDIR, tids[1])
self.assertEqual(gitstatus.locked_global(BASEDIR), False)
def file_edit(collection_path, file_id, git_name, git_mail):
"""The time-consuming parts of file-edit.
@param collection_path: str Absolute path to collection
@param file_id: str
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
"""
logger.debug('file_edit(%s,%s,%s,%s)' % (git_name, git_mail, collection_path, file_id))
fidentifier = Identifier(id=file_id)
file_ = DDRFile.from_identifier(fidentifier)
gitstatus.lock(settings.MEDIA_BASE, 'file_edit')
exit,status = file_.save(git_name, git_mail)
gitstatus_update.apply_async((collection_path,), countdown=2)
return status,collection_path,file_id
def collection_sync( git_name, git_mail, collection_path ):
"""Synchronizes collection repo with workbench server.
@param src_path: Absolute path to collection repo.
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@return collection_path: Absolute path to collection.
"""
gitstatus.lock(settings.MEDIA_BASE, 'collection_sync')
exit,status = sync(git_name, git_mail, collection_path)
# update search index
path = os.path.join(collection_path, 'collection.json')
with open(path, 'r') as f:
document = json.loads(f.read())
docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document)
return collection_path
def collection_save(collection_path, updated_files, git_name, git_mail):
"""The time-consuming parts of collection-edit.
@param collection_path: str Absolute path to collection
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
"""
logger.debug('collection_save(%s,%s,%s,%s)' % (
git_name, git_mail, collection_path, updated_files))
collection = Collection.from_json(collection_path)
gitstatus.lock(settings.MEDIA_BASE, 'collection_edit')
exit,status = collection.save(updated_files, git_name, git_mail)
gitstatus_update.apply_async((collection_path,), countdown=2)
return status,collection_path
def entity_newexpert(collection_path, entity_id, git_name, git_mail):
"""Create new entity using known entity ID.
@param collection_path: str Absolute path to collection
@param entity_id: str
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
"""
logger.debug('collection_entity_newexpert(%s,%s,%s,%s)' % (
collection_path, entity_id, git_name, git_mail))
collection = Collection.from_identifier(Identifier(path=collection_path))
gitstatus.lock(settings.MEDIA_BASE, 'entity_newexpert')
entity = Entity.create(collection, entity_id, git_name, git_mail)
gitstatus_update.apply_async((collection.path,), countdown=2)
return 'status',collection_path,entity.id
def file_edit(collection_path, file_id, git_name, git_mail):
"""The time-consuming parts of file-edit.
@param collection_path: str Absolute path to collection
@param file_id: str
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
"""
logger.debug('file_edit(%s,%s,%s,%s)' % (git_name, git_mail, collection_path, file_id))
model,repo,org,cid,eid,role,sha1 = models.split_object_id(file_id)
entity = Entity.from_json(Entity.entity_path(None,repo,org,cid,eid))
file_ = entity.file(repo, org, cid, eid, role, sha1)
gitstatus.lock(settings.MEDIA_BASE, 'file_edit')
exit,status = file_.save(git_name, git_mail)
gitstatus_update.apply_async((collection_path,), countdown=2)
return status,collection_path,file_id
def reload_files(collection_path, entity_id, git_name, git_mail, agent=''):
"""Regenerate entity.json's list of child files.
@param collection_path: string
@param entity_id: string
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
logger.debug('tasks.entity.reload_files(%s,%s,%s,%s,%s)' % (collection_path, entity_id, git_name, git_mail, agent))
gitstatus.lock(settings.MEDIA_BASE, 'reload_files')
entity = Entity.from_identifier(Identifier(entity_id))
collection = Collection.from_identifier(Identifier(path=collection_path))
exit,status,updated_files = entity.save(
git_name, git_mail,
collection,
{}
)
return status,collection_path,entity_id
def collection_sync( git_name, git_mail, collection_path ):
"""Synchronizes collection repo with workbench server.
@param src_path: Absolute path to collection repo.
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@return collection_path: Absolute path to collection.
"""
gitstatus.lock(settings.MEDIA_BASE, 'collection_sync')
collection = Collection.from_identifier(Identifier(path=collection_path))
exit,status = commands.sync(
git_name, git_mail,
collection
)
# update search index
collection = Collection.from_identifier(Identifier(path=collection_path))
try:
collection.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
except ConnectionError:
logger.error('Could not update search index')
return collection_path
def delete(collection, entity, git_name, git_mail, agent):
"""
@param collection_path: string
@param entity_id: string
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
logger.debug('tasks.entity.delete(%s,%s,%s,%s,%s)' % (
collection, entity, git_name, git_mail, agent
))
gitstatus.lock(settings.MEDIA_BASE, 'delete_entity')
status,message = entity.delete(git_name, git_mail, agent, commit=True)
logger.debug('Updating Elasticsearch')
if settings.DOCSTORE_ENABLED:
ds = docstore.Docstore()
try:
ds.delete(entity.id)
except ConnectionError:
logger.error('Could not delete document from Elasticsearch.')
return status,message,collection.path_abs,entity.id
def entity_edit(collection_path, entity_id, updated_files, git_name, git_mail, agent=''):
"""The time-consuming parts of entity-edit.
@param collection_path: str Absolute path to collection
@param entity_id: str
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
logger.debug('collection_entity_edit(%s,%s,%s,%s,%s)' % (
git_name, git_mail, collection_path, entity_id, agent))
collection = Collection.from_json(collection_path)
repo,org,cid,eid = entity_id.split('-')
entity_path = Entity.entity_path(None,repo,org,cid,eid)
entity = Entity.from_json(entity_path)
gitstatus.lock(settings.MEDIA_BASE, 'entity_edit')
exit,status = entity.save_part2(updated_files, collection, git_name, git_mail)
gitstatus_update.apply_async((collection.path,), countdown=2)
return status,collection_path,entity_id
def test_unlock(self):
os.makedirs(os.path.join(BASEDIR, 'tmp'))
lock_path = os.path.join(BASEDIR, 'tmp', 'gitstatus-lock')
before = os.path.exists(lock_path)
self.assertEqual(before, False)
# lock twice
tids = ['1234', '1248']
lockfile0 = gitstatus.lock(BASEDIR, tids[0])
after0 = os.path.exists(lock_path)
lockfile1 = gitstatus.lock(BASEDIR, tids[1])
after1 = os.path.exists(lock_path)
self.assertEqual(after0, True)
self.assertEqual(after1, True)
self.assertEqual(len(lockfile1.strip().split('\n')), 2)
# unlock
unlocked0 = gitstatus.unlock(BASEDIR, tids[0])
unlocked1 = gitstatus.unlock(BASEDIR, tids[1])
self.assertEqual(len(unlocked0.strip().split('\n')), 1)
self.assertEqual(unlocked1, '')
def entity_add_access( git_name, git_mail, entity, ddrfile, agent='' ):
"""
@param entity: Entity
@param ddrfile: DDRFile
@param src_path: Absolute path to an uploadable file.
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
gitstatus.lock(settings.MEDIA_BASE, 'entity_add_access')
file_,repo,log,op = entity.add_access(ddrfile, git_name, git_mail, agent)
if op and (op == 'pass'):
log.ok('Things are okay as they are. Leaving them alone.')
return file_.__dict__
file_,repo,log = entity.add_file_commit(file_, repo, log, git_name, git_mail, agent)
try:
file_.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
except ConnectionError:
log.not_ok('Could not post to Elasticsearch.')
return {
'id': file_.id,
'status': 'ok'
}
def entity_add_file( git_name, git_mail, entity, src_path, role, data, agent='' ):
"""
@param entity: Entity
@param src_path: Absolute path to an uploadable file.
@param role: Keyword of a file role.
@param data: Dict containing form data.
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
gitstatus.lock(settings.MEDIA_BASE, 'entity_add_file')
file_,repo,log = entity.add_file(src_path, role, data, git_name, git_mail, agent)
file_,repo,log = entity.add_file_commit(file_, repo, log, git_name, git_mail, agent)
log.ok('Updating Elasticsearch')
try:
result = file_.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
log.ok('| %s' % result)
except ConnectionError:
log.not_ok('Could not post to Elasticsearch.')
return {
'id': file_.id,
'status': 'ok'
}
def delete_entity( git_name, git_mail, collection_path, entity_id, agent='' ):
"""
@param collection_path: string
@param entity_id: string
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
gitstatus.lock(settings.MEDIA_BASE, 'delete_entity')
logger.debug('collection_delete_entity(%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, agent))
# remove the entity
collection = Collection.from_identifier(Identifier(collection_path))
entity = Entity.from_identifier(Identifier(entity_id))
status,message = commands.entity_destroy(
git_name, git_mail,
collection, entity,
agent
)
# update search index
try:
docstore.delete(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, entity_id)
except ConnectionError:
logger.error('Could not delete document from Elasticsearch.')
return status,message,collection_path,entity_id
def test_lock(self):
os.makedirs(os.path.join(BASEDIR, 'tmp'))
lock_path = os.path.join(BASEDIR, 'tmp', 'gitstatus-lock')
# no lockfile
lockfile_before = os.path.exists(lock_path)
before = os.path.exists(lock_path)
self.assertEqual(before, False)
self.assertEqual(lockfile_before, False)
tids = ['1234', '1248']
# '2014-07-15T15:17:15:254884 1234'
lockfile_after = gitstatus.lock(BASEDIR, tids[0])
after = os.path.exists(lock_path)
self.assertEqual(after, True)
self.assertIsNotNone(lockfile_after)
# check if properly formed
ts1,msg1 = lockfile_after.strip().split()
timestamp1 = datetime.strptime(ts1, '%Y-%m-%dT%H:%M:%S:%f')
self.assertEqual(isinstance(timestamp1, datetime), True)
self.assertEqual(msg1, tids[0])
# '2014-07-15T15:17:15:254884 1234\n2014-07-15T15:17:15:254907 1248'
lockfile_again = gitstatus.lock(BASEDIR, tids[1])
again = os.path.exists(lock_path)
self.assertEqual(again, True)
self.assertIsNotNone(lockfile_again)
# check if properly formed
lines = lockfile_again.strip().split('\n')
self.assertEqual(len(lines), 2)
for n,line in enumerate(lines):
ts2,msg2 = line.strip().split()
timestamp2 = datetime.strptime(ts2, '%Y-%m-%dT%H:%M:%S:%f')
self.assertEqual(isinstance(timestamp2, datetime), True)
self.assertEqual(msg2, tids[n])