コード例 #1
0
ファイル: models.py プロジェクト: densho/bkup-ddr-local
    def create(collection, entity_id, git_name, git_mail, agent=settings.AGENT):
        """create new entity given an entity ID
        """
        repo,org,cid,eid = entity_id.split('-')
        entity_path = Entity.entity_path(None, repo, org, cid, eid)
        
        # write entity.json template to entity location and commit
        Entity(entity_path).dump_json(path=settings.TEMPLATE_EJSON, template=True)
        exit,status = commands.entity_create(
            git_name, git_mail,
            collection.path, entity_id,
            [collection.json_path_rel, collection.ead_path_rel],
            [settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
            agent=agent)
        
        # load new entity, inherit values from parent, write and commit
        entity = Entity.from_json(entity_path)
        entity.inherit(collection)
        entity.dump_json()
        updated_files = [entity.json_path]
        exit,status = commands.entity_update(
            git_name, git_mail,
            collection.path, entity.id,
            updated_files,
            agent=agent)

        # delete cache, update search index
        collection.cache_delete()
        with open(entity.json_path, 'r') as f:
            document = json.loads(f.read())
        docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document)
        
        return entity
コード例 #2
0
ファイル: entity.py プロジェクト: denshoproject/ddr-cmdln
 def create(identifier, git_name, git_mail, agent='cmdln'):
     """Creates new Entity, writes files, performs initial commit
     
     @param identifier: Identifier
     @param git_name: str
     @param git_mail: str
     @param agent: str
     @returns: exit,status int,str
     """
     collection = identifier.collection().object()
     if not collection:
         raise Exception('Parent collection for %s does not exist.' %
                         identifier)
     exit, status = commands.entity_create(
         user_name=git_name,
         user_mail=git_mail,
         collection=collection,
         eidentifier=identifier,
         updated_files=[collection.json_path_rel, collection.ead_path_rel],
         agent=agent)
     if exit:
         raise Exception('Could not create new Entity: %s, %s' %
                         (exit, status))
     # load Entity object, inherit values from parent, write back to file
     entity = Entity.from_identifier(identifier)
     entity.inherit(collection)
     entity.write_json()
     updated_files = [entity.json_path]
     exit, status = commands.entity_update(git_name,
                                           git_mail,
                                           collection,
                                           entity,
                                           updated_files,
                                           agent=agent)
     return exit, status
コード例 #3
0
def test_10_entity_create(tmpdir, test_paths):
    """Create new entity in the collection

    collection = identifier.Identifier(test_paths['TEST_COLLECTION']).object()
    collection.notes = 'testing testing'
    collection.write_json()
    updated_files = [
        collection.identifier.path_abs('json')
    ]
    """
    collection = identifier.Identifier(test_paths['TEST_COLLECTION']).object()
    print(collection)
    for eid in TEST_EIDS:
        ei = identifier.Identifier(eid, collection.identifier.basepath)
        entity = models.entity.Entity.new(ei)
        print(entity)
        exit,status = commands.entity_create(
            GIT_USER, GIT_MAIL,
            collection,
            entity.identifier,
            [
                collection.json_path_rel,
                collection.ead_path_rel,
            ],
            agent=AGENT
        )
        print(exit,status)

    # confirm entity files exist
    assert os.path.exists(test_paths['COLLECTION_FILES'])
    for eid in TEST_EIDS:
        entity_path = os.path.join(test_paths['COLLECTION_FILES'],eid)
        assert os.path.exists(entity_path)
        assert os.path.exists(os.path.join(entity_path,'changelog'))
        assert os.path.exists(os.path.join(entity_path,'control'))
        assert os.path.exists(os.path.join(entity_path,'mets.xml'))
    # TODO test contents of entity files
    print('entity files exist')
    
    # confirm entities in changelog
    changelog_entries = []
    for eid in TEST_EIDS:
        changelog_entries.append('* Initialized entity {}'.format(eid))
    changelog = None
    with open(test_paths['COLLECTION_CHANGELOG'],'r') as cl:
        changelog = cl.read()
    for entry in changelog_entries:
        assert entry in changelog
    print('entities in changelog')
    
    # confirm entities in ead.xml
    assert os.path.exists(test_paths['COLLECTION_EAD'])
コード例 #4
0
    def new(identifier, git_name, git_mail, agent='cmdln'):
        """Creates new File (metadata only!), writes to filesystem, performs initial commit
        
        @param identifier: Identifier
        @param git_name: str
        @param git_mail: str
        @param agent: str
        @returns: exit,status int,str
        """
        parent = identifier.parent().object()
        if not parent:
            raise Exception('Parent for %s does not exist.' % identifier)
        file_ = File.create(identifier.path_abs(), identifier)
        file_.write_json()

        entity_file_edit(request, collection, file_, git_name, git_mail)

        exit, status = commands.entity_create(
            git_name,
            git_mail,
            collection,
            entity.identifier,
            [collection.json_path_rel, collection.ead_path_rel],
            [config.TEMPLATE_EJSON, config.TEMPLATE_METS],
            agent=agent)
        if exit:
            raise Exception('Could not create new Entity: %s, %s' %
                            (exit, status))
        # load Entity object, inherit values from parent, write back to file
        entity = Identifier(identifier).object()
        entity.inherit(collection)
        entity.write_json()
        updated_files = [entity.json_path]
        exit, status = commands.entity_update(git_name,
                                              git_mail,
                                              collection,
                                              entity,
                                              updated_files,
                                              agent=agent)
        return exit, status
コード例 #5
0
ファイル: files.py プロジェクト: densho/ddr-cmdln
    def new(identifier, git_name, git_mail, agent='cmdln'):
        """Creates new File (metadata only!), writes to filesystem, performs initial commit
        
        @param identifier: Identifier
        @param git_name: str
        @param git_mail: str
        @param agent: str
        @returns: exit,status int,str
        """
        parent = identifier.parent().object()
        if not parent:
            raise Exception('Parent for %s does not exist.' % identifier)
        file_ = File.create(identifier)
        file_.write_json()
        
        entity_file_edit(request, collection, file_, git_name, git_mail)

        exit,status = commands.entity_create(
            git_name, git_mail,
            collection, entity.identifier,
            [collection.json_path_rel, collection.ead_path_rel],
            [config.TEMPLATE_EJSON, config.TEMPLATE_METS],
            agent=agent
        )
        if exit:
            raise Exception('Could not create new Entity: %s, %s' % (exit, status))
        # load Entity object, inherit values from parent, write back to file
        entity = Identifier(identifier).object()
        entity.inherit(collection)
        entity.write_json()
        updated_files = [entity.json_path]
        exit,status = commands.entity_update(
            git_name, git_mail,
            collection, entity,
            updated_files,
            agent=agent
        )
        return exit,status
コード例 #6
0
ファイル: entity.py プロジェクト: raux/ddr-cmdln
 def new(identifier, git_name, git_mail, agent='cmdln'):
     """Creates new Entity, writes to filesystem, does initial commit.
     
     @param identifier: Identifier
     @param git_name: str
     @param git_mail: str
     @param agent: str
     @returns: exit,status int,str
     """
     collection = identifier.collection().object()
     if not collection:
         raise Exception('Parent collection for %s does not exist.' % identifier)
     entity = Entity.create(identifier.path_abs(), identifier)
     fileio.write_text(
         entity.dump_json(template=True),
         config.TEMPLATE_EJSON
     )
     exit,status = commands.entity_create(
         git_name, git_mail,
         collection, entity.identifier,
         [collection.json_path_rel, collection.ead_path_rel],
         [config.TEMPLATE_EJSON, config.TEMPLATE_METS],
         agent=agent
     )
     if exit:
         raise Exception('Could not create new Entity: %s, %s' % (exit, status))
     # load Entity object, inherit values from parent, write back to file
     entity = Entity.from_identifier(identifier)
     entity.inherit(collection)
     entity.write_json()
     updated_files = [entity.json_path]
     exit,status = commands.entity_update(
         git_name, git_mail,
         collection, entity,
         updated_files,
         agent=agent
     )
     return exit,status
コード例 #7
0
    def create(collection, entity_id, git_name, git_mail, agent=settings.AGENT):
        """create new entity given an entity ID
        TODO remove write and commit, just create object
        """
        eidentifier = Identifier(id=entity_id)
        entity_path = eidentifier.path_abs()
        
        # write entity.json template to entity location and commit
        fileio.write_text(Entity(entity_path).dump_json(template=True),
                   settings.TEMPLATE_EJSON)
        exit,status = commands.entity_create(
            git_name, git_mail,
            collection, eidentifier,
            [collection.json_path_rel, collection.ead_path_rel],
            [settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
            agent=agent)
        
        # load new entity, inherit values from parent, write and commit
        entity = Entity.from_json(entity_path)
        entity.inherit(collection)
        entity.write_json()
        updated_files = [entity.json_path]
        exit,status = commands.entity_update(
            git_name, git_mail,
            collection, entity,
            updated_files,
            agent=agent)

        # delete cache, update search index
        collection.cache_delete()
        with open(entity.json_path, 'r') as f:
            document = json.loads(f.read())
        try:
            docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document)
        except ConnectionError:
            logger.error('Could not post to Elasticsearch.')
        
        return entity
コード例 #8
0
ファイル: entity.py プロジェクト: densho/ddr-cmdln
 def new(identifier, git_name, git_mail, agent='cmdln'):
     """Creates new Entity, writes to filesystem, does initial commit.
     
     @param identifier: Identifier
     @param git_name: str
     @param git_mail: str
     @param agent: str
     @returns: exit,status int,str
     """
     collection = identifier.collection().object()
     if not collection:
         raise Exception('Parent collection for %s does not exist.' % identifier)
     entity = Entity.create(identifier)
     fileio.write_text(
         entity.dump_json(template=True),
         config.TEMPLATE_EJSON
     )
     exit,status = commands.entity_create(
         git_name, git_mail,
         collection, entity.identifier,
         [collection.json_path_rel, collection.ead_path_rel],
         [config.TEMPLATE_EJSON, config.TEMPLATE_METS],
         agent=agent
     )
     if exit:
         raise Exception('Could not create new Entity: %s, %s' % (exit, status))
     # load Entity object, inherit values from parent, write back to file
     entity = Entity.from_identifier(identifier)
     entity.inherit(collection)
     entity.write_json()
     updated_files = [entity.json_path]
     exit,status = commands.entity_update(
         git_name, git_mail,
         collection, entity,
         updated_files,
         agent=agent
     )
     return exit,status
コード例 #9
0
ファイル: densho.py プロジェクト: densho/ddr-local
def import_entities(csv_path, collection_path, git_name, git_mail):
    """
    @param csv_path: Absolute path to CSV data file.
    @param collection_path: Absolute path to collection repo.
    @param git_name: Username for use in changelog, git log
    @param git_mail: User email address for use in changelog, git log
    """
    rows = read_csv(csv_path)
    headers = rows[0]
    rows = rows[1:]

    headers = replace_variant_headers("entity", headers)
    field_names = FIELD_NAMES["entity"]
    nonrequired_fields = REQUIRED_FIELDS_EXCEPTIONS["entity"]
    required_fields = get_required_fields(ENTITY_FIELDS, nonrequired_fields)
    rows = replace_variant_cv_field_values("entity", headers, rows)
    invalid_headers("entity", headers, field_names, nonrequired_fields)
    all_rows_valid("entity", headers, required_fields, rows)
    if True:
        collection = Collection.from_json(collection_path)
        print(collection)

        # --------------------------------------------------
        def prep_creators(data):
            return [x.strip() for x in data.strip().split(";") if x]

        def prep_language(data):
            """language can be 'eng', 'eng;jpn', 'eng:English', 'jpn:Japanese'
            """
            y = []
            for x in data.strip().split(";"):
                if ":" in x:
                    y.append(x.strip().split(":")[0])
                else:
                    y.append(x.strip())
            return y

        def prep_topics(data):
            return [x.strip() for x in data.strip().split(";") if x]

        def prep_persons(data):
            return [x.strip() for x in data.strip().split(";") if x]

        def prep_facility(data):
            return [x.strip() for x in data.strip().split(";") if x]

        # --------------------------------------------------

        print("Data file looks ok")
        started = datetime.now()
        print("%s starting import" % dtfmt(started))
        print("")
        for n, row in enumerate(rows):
            rowstarted = datetime.now()
            rowd = make_row_dict(headers, row)

            # create new entity
            entity_id = rowd["id"]
            entity_path = os.path.join(collection_path, COLLECTION_FILES_PREFIX, entity_id)

            # write entity.json template to entity location
            fileio.write_text(Entity(entity_path).dump_json(template=True), TEMPLATE_EJSON)
            # commit files
            exit, status = commands.entity_create(
                git_name,
                git_mail,
                collection.path,
                entity_id,
                [collection.json_path_rel, collection.ead_path_rel],
                [TEMPLATE_EJSON, TEMPLATE_METS],
                agent=AGENT,
            )

            # reload newly-created Entity object
            entity = Entity.from_json(entity_path)

            # preppers
            rowd["creators"] = prep_creators(rowd["creators"])
            rowd["language"] = prep_language(rowd["language"])
            rowd["topics"] = prep_topics(rowd["topics"])
            rowd["persons"] = prep_persons(rowd["persons"])
            rowd["facility"] = prep_facility(rowd["facility"])

            # insert values from CSV
            for key in rowd.keys():
                setattr(entity, key, rowd[key])
            entity.record_created = datetime.now()
            entity.record_lastmod = datetime.now()

            # write back to file
            entity.write_json()
            updated_files = [entity.json_path]
            exit, status = commands.entity_update(
                git_name, git_mail, entity.parent_path, entity.id, updated_files, agent=AGENT
            )

            rowfinished = datetime.now()
            rowelapsed = rowfinished - rowstarted
            print("%s %s/%s %s (%s)" % (dtfmt(rowfinished), n + 1, len(rows), entity.id, rowelapsed))
        finished = datetime.now()
        elapsed = finished - started
        print("")
        print("%s done (%s rows)" % (dtfmt(finished), len(rows)))
        print("%s elapsed" % elapsed)
        print("")
コード例 #10
0
def new( request, repo, org, cid ):
    """Gets new EID from workbench, creates new entity record.
    
    If it messes up, goes back to collection.
    """
    git_name = request.session.get('git_name')
    git_mail = request.session.get('git_mail')
    if not (git_name and git_mail):
        messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
    collection = Collection.from_request(request)
    if collection.locked():
        messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
        return HttpResponseRedirect(collection.absolute_url())
    collection.repo_fetch()
    if collection.repo_behind():
        messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
        return HttpResponseRedirect(collection.absolute_url())
    
    ic = idservice.IDServiceClient()
    # resume session
    auth_status,auth_reason = ic.resume(request.session['idservice_token'])
    if auth_status != 200:
        request.session['idservice_username'] = None
        request.session['idservice_token'] = None
        messages.warning(
            request,
            'Session resume failed: %s %s (%s)' % (
                auth_status,auth_reason,settings.IDSERVICE_API_BASE
            )
        )
        return HttpResponseRedirect(collection.absolute_url())
    # get new entity ID
    http_status,http_reason,new_entity_id = ic.next_object_id(
        collection.identifier,
        'entity'
    )
    if http_status not in [200,201]:
        err = '%s %s' % (http_status, http_reason)
        msg = WEBUI_MESSAGES['VIEWS_ENT_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err)
        logger.error(msg)
        messages.error(request, msg)
        return HttpResponseRedirect(collection.absolute_url())
    
    eidentifier = Identifier(id=new_entity_id)
    # create new entity
    entity_path = eidentifier.path_abs()
    # write entity.json template to entity location
    fileio.write_text(
        Entity(entity_path).dump_json(template=True),
        settings.TEMPLATE_EJSON
    )
    
    # commit files
    exit,status = commands.entity_create(
        git_name, git_mail,
        collection, eidentifier,
        [collection.json_path_rel, collection.ead_path_rel],
        [settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
        agent=settings.AGENT
    )
    
    # load Entity object, inherit values from parent, write back to file
    entity = Entity.from_identifier(eidentifier)
    entity.inherit(collection)
    entity.write_json()
    updated_files = [entity.json_path]
    exit,status = commands.entity_update(
        git_name, git_mail,
        collection, entity,
        updated_files,
        agent=settings.AGENT
    )
    
    collection.cache_delete()
    if exit:
        logger.error(exit)
        logger.error(status)
        messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
    else:
        # update search index
        try:
            entity.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
        except ConnectionError:
            logger.error('Could not post to Elasticsearch.')
        gitstatus_update.apply_async((collection.path,), countdown=2)
        # positive feedback
        return HttpResponseRedirect(reverse('webui-entity-edit', args=entity.idparts))
    
    # something happened...
    logger.error('Could not create new entity!')
    messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_ERR_CREATE'])
    return HttpResponseRedirect(collection.absolute_url())