コード例 #1
0
ファイル: collection.py プロジェクト: densho/ddr-local
def save(collection_path, cleaned_data, git_name, git_mail):
    """The time-consuming parts of collection-edit.
    
    @param collection_path: str Absolute path to collection
    @param cleaned_data: dict form.cleaned_data
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    """
    logger.debug('tasks.collection.save(%s,%s,%s)' % (
        git_name, git_mail, collection_path))
    
    collection = Collection.from_identifier(Identifier(path=collection_path))
    gitstatus.lock(settings.MEDIA_BASE, 'collection_edit')
    
    exit,status,updated_files = collection.save(
        git_name, git_mail,
        cleaned_data
    )
    
    dvcs_tasks.gitstatus_update.apply_async(
        (collection_path,),
        countdown=2
    )
    
    return status,collection_path
コード例 #2
0
def reindex( index ):
    """
    @param index: Name of index to create or update
    """
    gitstatus.lock(settings.MEDIA_BASE, 'reindex')
    logger.debug('------------------------------------------------------------------------')
    logger.debug('webui.tasks.reindex(%s)' % index)
    statuses = []
    if not os.path.exists(settings.MEDIA_BASE):
        raise NameError('MEDIA_BASE does not exist - you need to remount!')
    logger.debug('webui.tasks.reindex(%s)' % index)
    logger.debug('DOCSTORE_HOSTS: %s' % settings.DOCSTORE_HOSTS)
    logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
    logger.debug('deleting existing index: %s' % index)
    delete_status = docstore.delete_index(settings.DOCSTORE_HOSTS, index)
    logger.debug(delete_status)
    logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
    logger.debug('creating new index: %s' % index)
    create_status = docstore.create_index(settings.DOCSTORE_HOSTS, index)
    logger.debug(create_status)
    logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
    logger.debug('mappings: %s, %s' % (docstore.HARD_CODED_MAPPINGS_PATH, models.MODELS_DIR))
    mappings_status = docstore.put_mappings(settings.DOCSTORE_HOSTS, index,
                                            docstore.HARD_CODED_MAPPINGS_PATH, models.MODELS_DIR)
    logger.debug(mappings_status)
    logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
    logger.debug('facets')
    facets_status = docstore.put_facets(settings.DOCSTORE_HOSTS, index)
    logger.debug(facets_status)
    logger.debug('- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ')
    logger.debug('indexing')
    index_status = docstore.index(settings.DOCSTORE_HOSTS, index, path=settings.MEDIA_BASE,
                                  recursive=True, public=False)
    logger.debug(index_status)
    return statuses
コード例 #3
0
ファイル: collection.py プロジェクト: densho/ddr-local
def signatures(collection_path, git_name, git_mail):
    """Identifies signature files for collection and entities.
    
    @param collection_path: Absolute path to collection repo.
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @return collection_path: Absolute path to collection.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'collection_signatures')
    collection = Collection.from_identifier(Identifier(path=collection_path))
    updates = signatures.find_updates(collection)
    files_written = signatures.write_updates(updates)

    # TODO move this code to webui.models.Collection
    status,msg = signatures.commit_updates(
        collection,
        files_written,
        git_name, git_mail, agent='ddr-local'
    )
    logger.debug('DONE')
    logger.debug('Updating Elasticsearch')
    if settings.DOCSTORE_ENABLED:
        collection = Collection.from_identifier(Identifier(path=collection_path))
        try:
            collection.post_json()
        except ConnectionError:
            logger.error('Could not update search index')
    
    return collection_path
コード例 #4
0
ファイル: collection.py プロジェクト: densho/ddr-local
def sync( git_name, git_mail, collection_path ):
    """Synchronizes collection repo with workbench server.
    
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param collection_path: Absolute path to collection repo.
    @return collection_path: Absolute path to collection.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'collection_sync')
    ci = Identifier(path=collection_path)
    collection = Collection.from_identifier(ci)
    
    # TODO move this code to webui.models.Collection.sync
    exit,status = commands.sync(
        git_name, git_mail,
        collection
    )
    logger.debug('Updating Elasticsearch')
    if settings.DOCSTORE_ENABLED:
        try:
            collection.reindex()
        except ConnectionError:
            logger.error('Could not update search index')
    
    return collection_path
コード例 #5
0
ファイル: entity.py プロジェクト: densho/ddr-local
def entity_edit(collection_path, entity_id, form_data, git_name, git_mail, agent=''):
    """The time-consuming parts of entity-edit.
    
    @param collection_path: str Absolute path to collection
    @param entity_id: str
    @param form_data: dict
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    logger.debug('tasks.entity.entity_edit(%s,%s,%s,%s,%s)' % (
        git_name, git_mail, collection_path, entity_id, agent))
    collection = Collection.from_identifier(Identifier(path=collection_path))
    entity = Entity.from_identifier(Identifier(id=entity_id))
    gitstatus.lock(settings.MEDIA_BASE, 'entity_edit')
    
    exit,status,updated_files = entity.save(
        git_name, git_mail,
        collection,
        form_data
    )
    
    dvcs_tasks.gitstatus_update.apply_async(
        (collection.path,),
        countdown=2
    )
    return status,collection_path,entity_id
コード例 #6
0
def delete_file( git_name, git_mail, collection_path, entity_id, file_basename, agent='' ):
    """
    @param collection_path: string
    @param entity_id: string
    @param file_basename: string
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    logger.debug('delete_file(%s,%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, file_basename, agent))
    gitstatus.lock(settings.MEDIA_BASE, 'delete_file')
    file_id = os.path.splitext(file_basename)[0]
    file_ = DDRFile.from_identifier(Identifier(file_id))
    entity = Entity.from_identifier(Identifier(entity_id))
    collection = Collection.from_identifier(Identifier(path=collection_path))
    logger.debug('delete from repository')
    rm_files,updated_files = entity.prep_rm_file(file_)
    status,message = commands.file_destroy(
        git_name, git_mail,
        collection, entity,
        rm_files, updated_files,
        agent
    )
    logger.debug('delete from search index')
    try:
        docstore.delete(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, file_.id)
    except ConnectionError:
        logger.error('Could not delete document from Elasticsearch.')
    return status,message,collection_path,file_basename
コード例 #7
0
ファイル: tasks.py プロジェクト: densho/bkup-ddr-local
def delete_file( git_name, git_mail, collection_path, entity_id, file_basename, agent='' ):
    """
    @param collection_path: string
    @param entity_id: string
    @param file_basename: string
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    logger.debug('delete_file(%s,%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, file_basename, agent))
    gitstatus.lock(settings.MEDIA_BASE, 'delete_file')
    # TODO rm_files list should come from the File model
    file_id = os.path.splitext(file_basename)[0]
    repo,org,cid,eid,role,sha1 = file_id.split('-')
    entity = Entity.from_json(Entity.entity_path(None,repo,org,cid,eid))
    file_ = entity.file(repo, org, cid, eid, role, sha1)
    rm_files = file_.files_rel(collection_path)
    logger.debug('rm_files: %s' % rm_files)
    # remove file from entity.json
    # TODO move this to commands.file_destroy or models.Entity
    for f in entity.files:
        if f.basename == file_basename:
            entity.files.remove(f)
    entity.dump_json()
    updated_files = ['entity.json']
    logger.debug('updated_files: %s' % updated_files)
    status,message = file_destroy(git_name, git_mail, collection_path, entity_id, rm_files, updated_files, agent)
    return status,message,collection_path,file_basename
コード例 #8
0
ファイル: tasks.py プロジェクト: densho/bkup-ddr-local
def entity_add_access( git_name, git_mail, entity, ddrfile, agent='' ):
    """
    @param entity: Entity
    @param ddrfile: DDRFile
    @param src_path: Absolute path to an uploadable file.
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'entity_add_access')
    return entity.add_access(git_name, git_mail, ddrfile, agent)
コード例 #9
0
ファイル: tasks.py プロジェクト: densho/bkup-ddr-local
def delete_entity( git_name, git_mail, collection_path, entity_id, agent='' ):
    """
    @param collection_path: string
    @param entity_id: string
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'delete_entity')
    logger.debug('collection_delete_entity(%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, agent))
    status,message = entity_destroy(git_name, git_mail, collection_path, entity_id, agent)
    return status,message,collection_path,entity_id
コード例 #10
0
ファイル: tasks.py プロジェクト: densho/bkup-ddr-local
def entity_add_file( git_name, git_mail, entity, src_path, role, data, agent='' ):
    """
    @param entity: Entity
    @param src_path: Absolute path to an uploadable file.
    @param role: Keyword of a file role.
    @param data: Dict containing form data.
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'entity_add_file')
    return entity.add_file(git_name, git_mail, src_path, role, data, agent)
コード例 #11
0
 def test_locked_global(self):
     os.makedirs(os.path.join(BASEDIR, 'tmp'))
     lock_path = os.path.join(BASEDIR, 'tmp', 'gitstatus-lock')
     self.assertEqual(gitstatus.locked_global(BASEDIR), False)
     # lock
     tids = ['1234', '1248']
     lockfile0 = gitstatus.lock(BASEDIR, tids[0])
     self.assertNotEqual(gitstatus.locked_global(BASEDIR), False)
     lockfile1 = gitstatus.lock(BASEDIR, tids[1])
     self.assertNotEqual(gitstatus.locked_global(BASEDIR), False)
     # unlock
     unlocked0 = gitstatus.unlock(BASEDIR, tids[0])
     self.assertNotEqual(gitstatus.locked_global(BASEDIR), False)
     unlocked1 = gitstatus.unlock(BASEDIR, tids[1])
     self.assertEqual(gitstatus.locked_global(BASEDIR), False)
コード例 #12
0
def file_edit(collection_path, file_id, git_name, git_mail):
    """The time-consuming parts of file-edit.
    
    @param collection_path: str Absolute path to collection
    @param file_id: str
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    """
    logger.debug('file_edit(%s,%s,%s,%s)' % (git_name, git_mail, collection_path, file_id))
    fidentifier = Identifier(id=file_id)
    file_ = DDRFile.from_identifier(fidentifier)
    gitstatus.lock(settings.MEDIA_BASE, 'file_edit')
    exit,status = file_.save(git_name, git_mail)
    gitstatus_update.apply_async((collection_path,), countdown=2)
    return status,collection_path,file_id
コード例 #13
0
ファイル: tasks.py プロジェクト: densho/bkup-ddr-local
def collection_sync( git_name, git_mail, collection_path ):
    """Synchronizes collection repo with workbench server.
    
    @param src_path: Absolute path to collection repo.
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @return collection_path: Absolute path to collection.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'collection_sync')
    exit,status = sync(git_name, git_mail, collection_path)
    # update search index
    path = os.path.join(collection_path, 'collection.json')
    with open(path, 'r') as f:
        document = json.loads(f.read())
    docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document)
    return collection_path
コード例 #14
0
ファイル: tasks.py プロジェクト: densho/bkup-ddr-local
def collection_save(collection_path, updated_files, git_name, git_mail):
    """The time-consuming parts of collection-edit.
    
    @param collection_path: str Absolute path to collection
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    """
    logger.debug('collection_save(%s,%s,%s,%s)' % (
        git_name, git_mail, collection_path, updated_files))
    
    collection = Collection.from_json(collection_path)
    
    gitstatus.lock(settings.MEDIA_BASE, 'collection_edit')
    exit,status = collection.save(updated_files, git_name, git_mail)
    gitstatus_update.apply_async((collection_path,), countdown=2)
    
    return status,collection_path
コード例 #15
0
def entity_newexpert(collection_path, entity_id, git_name, git_mail):
    """Create new entity using known entity ID.
    
    @param collection_path: str Absolute path to collection
    @param entity_id: str
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    """
    logger.debug('collection_entity_newexpert(%s,%s,%s,%s)' % (
        collection_path, entity_id, git_name, git_mail))
    
    collection = Collection.from_identifier(Identifier(path=collection_path))
    
    gitstatus.lock(settings.MEDIA_BASE, 'entity_newexpert')
    entity = Entity.create(collection, entity_id, git_name, git_mail)
    gitstatus_update.apply_async((collection.path,), countdown=2)

    return 'status',collection_path,entity.id
コード例 #16
0
ファイル: tasks.py プロジェクト: densho/bkup-ddr-local
def file_edit(collection_path, file_id, git_name, git_mail):
    """The time-consuming parts of file-edit.
    
    @param collection_path: str Absolute path to collection
    @param file_id: str
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    """
    logger.debug('file_edit(%s,%s,%s,%s)' % (git_name, git_mail, collection_path, file_id))
    
    model,repo,org,cid,eid,role,sha1 = models.split_object_id(file_id)
    entity = Entity.from_json(Entity.entity_path(None,repo,org,cid,eid))
    file_ = entity.file(repo, org, cid, eid, role, sha1)
    
    gitstatus.lock(settings.MEDIA_BASE, 'file_edit')
    exit,status = file_.save(git_name, git_mail)
    gitstatus_update.apply_async((collection_path,), countdown=2)
    
    return status,collection_path,file_id
コード例 #17
0
ファイル: entity.py プロジェクト: densho/ddr-local
def reload_files(collection_path, entity_id, git_name, git_mail, agent=''):
    """Regenerate entity.json's list of child files.
    
    @param collection_path: string
    @param entity_id: string
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    logger.debug('tasks.entity.reload_files(%s,%s,%s,%s,%s)' % (collection_path, entity_id, git_name, git_mail, agent))
    gitstatus.lock(settings.MEDIA_BASE, 'reload_files')
    entity = Entity.from_identifier(Identifier(entity_id))
    collection = Collection.from_identifier(Identifier(path=collection_path))
    
    exit,status,updated_files = entity.save(
        git_name, git_mail,
        collection,
        {}
    )

    return status,collection_path,entity_id
コード例 #18
0
def collection_sync( git_name, git_mail, collection_path ):
    """Synchronizes collection repo with workbench server.
    
    @param src_path: Absolute path to collection repo.
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @return collection_path: Absolute path to collection.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'collection_sync')
    collection = Collection.from_identifier(Identifier(path=collection_path))
    exit,status = commands.sync(
        git_name, git_mail,
        collection
    )
    # update search index
    collection = Collection.from_identifier(Identifier(path=collection_path))
    try:
        collection.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
    except ConnectionError:
        logger.error('Could not update search index')
    return collection_path
コード例 #19
0
ファイル: entity.py プロジェクト: densho/ddr-local
def delete(collection, entity, git_name, git_mail, agent):
    """
    @param collection_path: string
    @param entity_id: string
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    logger.debug('tasks.entity.delete(%s,%s,%s,%s,%s)' % (
        collection, entity, git_name, git_mail, agent
    ))
    gitstatus.lock(settings.MEDIA_BASE, 'delete_entity')
    status,message = entity.delete(git_name, git_mail, agent, commit=True)
    logger.debug('Updating Elasticsearch')
    if settings.DOCSTORE_ENABLED:
        ds = docstore.Docstore()
        try:
            ds.delete(entity.id)
        except ConnectionError:
            logger.error('Could not delete document from Elasticsearch.')
    return status,message,collection.path_abs,entity.id
コード例 #20
0
ファイル: tasks.py プロジェクト: densho/bkup-ddr-local
def entity_edit(collection_path, entity_id, updated_files, git_name, git_mail, agent=''):
    """The time-consuming parts of entity-edit.
    
    @param collection_path: str Absolute path to collection
    @param entity_id: str
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    logger.debug('collection_entity_edit(%s,%s,%s,%s,%s)' % (
        git_name, git_mail, collection_path, entity_id, agent))
    
    collection = Collection.from_json(collection_path)
    repo,org,cid,eid = entity_id.split('-')
    entity_path = Entity.entity_path(None,repo,org,cid,eid)
    entity = Entity.from_json(entity_path)
    
    gitstatus.lock(settings.MEDIA_BASE, 'entity_edit')
    exit,status = entity.save_part2(updated_files, collection, git_name, git_mail)
    gitstatus_update.apply_async((collection.path,), countdown=2)

    return status,collection_path,entity_id
コード例 #21
0
 def test_unlock(self):
     os.makedirs(os.path.join(BASEDIR, 'tmp'))
     lock_path = os.path.join(BASEDIR, 'tmp', 'gitstatus-lock')
     
     before = os.path.exists(lock_path)
     self.assertEqual(before, False)
     
     # lock twice
     tids = ['1234', '1248']
     lockfile0 = gitstatus.lock(BASEDIR, tids[0])
     after0 = os.path.exists(lock_path)
     lockfile1 = gitstatus.lock(BASEDIR, tids[1])
     after1 = os.path.exists(lock_path)
     self.assertEqual(after0, True)
     self.assertEqual(after1, True)
     self.assertEqual(len(lockfile1.strip().split('\n')), 2)
     
     # unlock
     unlocked0 = gitstatus.unlock(BASEDIR, tids[0])
     unlocked1 = gitstatus.unlock(BASEDIR, tids[1])
     self.assertEqual(len(unlocked0.strip().split('\n')), 1)
     self.assertEqual(unlocked1, '')
コード例 #22
0
def entity_add_access( git_name, git_mail, entity, ddrfile, agent='' ):
    """
    @param entity: Entity
    @param ddrfile: DDRFile
    @param src_path: Absolute path to an uploadable file.
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'entity_add_access')
    file_,repo,log,op = entity.add_access(ddrfile, git_name, git_mail, agent)
    if op and (op == 'pass'):
        log.ok('Things are okay as they are.  Leaving them alone.')
        return file_.__dict__
    file_,repo,log = entity.add_file_commit(file_, repo, log, git_name, git_mail, agent)
    try:
        file_.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
    except ConnectionError:
        log.not_ok('Could not post to Elasticsearch.')
    return {
        'id': file_.id,
        'status': 'ok'
    }
コード例 #23
0
def entity_add_file( git_name, git_mail, entity, src_path, role, data, agent='' ):
    """
    @param entity: Entity
    @param src_path: Absolute path to an uploadable file.
    @param role: Keyword of a file role.
    @param data: Dict containing form data.
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'entity_add_file')
    file_,repo,log = entity.add_file(src_path, role, data, git_name, git_mail, agent)
    file_,repo,log = entity.add_file_commit(file_, repo, log, git_name, git_mail, agent)
    log.ok('Updating Elasticsearch')
    try:
        result = file_.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
        log.ok('| %s' % result)
    except ConnectionError:
        log.not_ok('Could not post to Elasticsearch.')
    return {
        'id': file_.id,
        'status': 'ok'
    }
コード例 #24
0
def delete_entity( git_name, git_mail, collection_path, entity_id, agent='' ):
    """
    @param collection_path: string
    @param entity_id: string
    @param git_name: Username of git committer.
    @param git_mail: Email of git committer.
    @param agent: (optional) Name of software making the change.
    """
    gitstatus.lock(settings.MEDIA_BASE, 'delete_entity')
    logger.debug('collection_delete_entity(%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, agent))
    # remove the entity
    collection = Collection.from_identifier(Identifier(collection_path))
    entity = Entity.from_identifier(Identifier(entity_id))
    status,message = commands.entity_destroy(
        git_name, git_mail,
        collection, entity,
        agent
    )
    # update search index
    try:
        docstore.delete(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, entity_id)
    except ConnectionError:
        logger.error('Could not delete document from Elasticsearch.')
    return status,message,collection_path,entity_id
コード例 #25
0
 def test_lock(self):
     os.makedirs(os.path.join(BASEDIR, 'tmp'))
     lock_path = os.path.join(BASEDIR, 'tmp', 'gitstatus-lock')
     
     # no lockfile
     lockfile_before = os.path.exists(lock_path)
     before = os.path.exists(lock_path)
     self.assertEqual(before, False)
     self.assertEqual(lockfile_before, False)
     
     tids = ['1234', '1248']
     
     # '2014-07-15T15:17:15:254884 1234'
     lockfile_after = gitstatus.lock(BASEDIR, tids[0])
     after = os.path.exists(lock_path)
     self.assertEqual(after, True)
     self.assertIsNotNone(lockfile_after)
     # check if properly formed
     ts1,msg1 = lockfile_after.strip().split()
     timestamp1 = datetime.strptime(ts1, '%Y-%m-%dT%H:%M:%S:%f')
     self.assertEqual(isinstance(timestamp1, datetime), True)
     self.assertEqual(msg1, tids[0])
     
     # '2014-07-15T15:17:15:254884 1234\n2014-07-15T15:17:15:254907 1248'
     lockfile_again = gitstatus.lock(BASEDIR, tids[1])
     again = os.path.exists(lock_path)
     self.assertEqual(again, True)
     self.assertIsNotNone(lockfile_again)
     # check if properly formed
     lines = lockfile_again.strip().split('\n')
     self.assertEqual(len(lines), 2)
     for n,line in enumerate(lines):
         ts2,msg2 = line.strip().split()
         timestamp2 = datetime.strptime(ts2, '%Y-%m-%dT%H:%M:%S:%f')
         self.assertEqual(isinstance(timestamp2, datetime), True)
         self.assertEqual(msg2, tids[n])