Esempio n. 1
0
    def create(collection_path, git_name, git_mail):
        """create new entity given an entity ID
        TODO remove write and commit, just create object
        """
        # write collection.json template to collection location and commit
        fileio.write_text(Collection(collection_path).dump_json(template=True),
                   settings.TEMPLATE_CJSON)
        templates = [settings.TEMPLATE_CJSON, settings.TEMPLATE_EAD]
        agent = settings.AGENT

        cidentifier = Identifier(collection_path)
        exit,status = commands.create(
            git_name, git_mail, cidentifier, templates, agent
        )
        
        collection = Collection.from_identifier(cidentifier)
        
        # [delete cache], update search index
        #collection.cache_delete()
        if settings.DOCSTORE_ENABLED:
            try:
                docstore.Docstore().post(self)
            except ConnectionError:
                logger.error('Could not post to Elasticsearch.')
        
        return collection
Esempio n. 2
0
def write_object_metadata(obj, tmp_dir, log):
    tmp_json = os.path.join(tmp_dir, os.path.basename(obj.json_path))
    log.ok('| %s' % tmp_json)
    fileio.write_text(obj.dump_json(), tmp_json)
    if not os.path.exists(tmp_json):
        log.crash('Could not write file metadata %s' % tmp_json)
    return tmp_json
Esempio n. 3
0
 def create( path, collection_id, entity_id ):
     logging.debug('    EntityControlFile.create({})'.format(path))
     t = load_template(ENTITY_CONTROL_TEMPLATE)
     fileio.write_text(
         t.format(cid=collection_id, eid=entity_id),
         path
     )
Esempio n. 4
0
def write_object_metadata(obj, tmp_dir, log):
    tmp_json = os.path.join(tmp_dir, os.path.basename(obj.json_path))
    log.ok('| %s' % tmp_json)
    fileio.write_text(obj.dump_json(), tmp_json)
    if not os.path.exists(tmp_json):
        log.crash('Could not write file metadata %s' % tmp_json)
    return tmp_json
Esempio n. 5
0
 def write_xml(self, path=None):
     """Write METS XML file to disk.
     
     @param path: str Alternate absolute file path
     """
     if not path:
         path = self.mets_path
     fileio.write_text(self.dump_xml(), path)
Esempio n. 6
0
 def write_json(self, obj_metadata={}):
     """Write Collection JSON file to disk.
     
     @param obj_metadata: dict Cached results of object_metadata.
     """
     if not os.path.exists(self.identifier.path_abs()):
         os.makedirs(self.identifier.path_abs())
     fileio.write_text(
         self.dump_json(doc_metadata=True, obj_metadata=obj_metadata),
         self.json_path)
Esempio n. 7
0
def test_write_text(tmpdir):
    # TODO test bad chars
    path = str(tmpdir / 'write_text.json')
    # regular
    fileio.write_text(TEXT, path)
    with open(path, 'r') as f:
        written = f.read()
    assert written == TEXT
    # clean up
    os.remove(path)
Esempio n. 8
0
 def write(self, path: str):
     """Write to the specified file (.json or .csv).
     
     @param path: Absolute path to file; must be .json or .csv.
     """
     extension = os.path.splitext(path)[1]
     if not extension in ['.json', '.csv']:
         raise Exception('Index.read only writes .json and .csv files.')
     if extension.lower() == '.json':
         fileio.write_text(self.dump_json(), path)
     elif extension.lower() == '.csv':
         fileio.write_text(self.dump_csv(), path)
Esempio n. 9
0
def edit_json( request, repo, org, cid, eid ):
    """
    NOTE: will permit editing even if entity is locked!
    (which you need to do sometimes).
    """
    entity = Entity.from_request(request)
    collection = entity.collection()
    #if collection.locked():
    #    messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
    #    return HttpResponseRedirect(entity.absolute_url())
    #collection.repo_fetch()
    #if collection.repo_behind():
    #    messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
    #    return HttpResponseRedirect(entity.absolute_url())
    #if entity.locked():
    #    messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
    #    return HttpResponseRedirect(entity.absolute_url())
    #
    if request.method == 'POST':
        form = JSONForm(request.POST)
        if form.is_valid():
            git_name = request.session.get('git_name')
            git_mail = request.session.get('git_mail')
            if git_name and git_mail:
                json_text = form.cleaned_data['json']
                fileio.write_text(json_text, entity.json_path)
                
                exit,status = commands.entity_update(
                    git_name, git_mail,
                    collection, entity,
                    [entity.json_path],
                    agent=settings.AGENT
                )
                
                collection.cache_delete()
                if exit:
                    messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
                else:
                    gitstatus_update.apply_async((collection.path,), countdown=2)
                    messages.success(request, WEBUI_MESSAGES['VIEWS_ENT_UPDATED'])
                    return HttpResponseRedirect(entity.absolute_url())
            else:
                messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
    else:
        form = JSONForm({'json': entity.dump_json(),})
    return render_to_response(
        'webui/entities/edit-raw.html',
        {'entity': entity,
         'form': form,},
        context_instance=RequestContext(request, processors=[])
    )
Esempio n. 10
0
def test_write_text():
    # TODO test bad chars
    path = os.path.join(TESTING_BASE_DIR, 'write_text.json')
    # regular
    fileio.write_text(TEXT, path)
    with open(path, 'r') as f:
        written = f.read()
    assert written == TEXT
    # utf8_strict
    fileio.write_text(TEXT, path, utf8_strict=True)
    with open(path, 'r') as f:
        written = f.read()
    assert written == TEXT
    # clean up
    os.remove(path)
Esempio n. 11
0
 def write_json(self, doc_metadata=True, obj_metadata={}, force=False):
     """Write Collection/Entity JSON file to disk.
     
     @param doc_metadata: boolean
     @param obj_metadata: dict Cached results of object_metadata.
     @param force: boolean Write even nothing looks changed.
     """
     if force or self.is_modified():
         if not os.path.exists(os.path.dirname(self.json_path)):
             os.makedirs(os.path.dirname(self.json_path))
         fileio.write_text(
             self.dump_json(
                 doc_metadata=doc_metadata, obj_metadata=obj_metadata
             ),
             self.json_path
         )
Esempio n. 12
0
 def new(identifier, git_name, git_mail, agent='cmdln'):
     """Creates new Collection, writes to filesystem, performs initial commit
     
     @param identifier: Identifier
     @param git_name: str
     @param git_mail: str
     @param agent: str
     @returns: exit,status int,str
     """
     collection = Collection.create(identifier.path_abs(), identifier)
     fileio.write_text(collection.dump_json(template=True),
                       config.TEMPLATE_CJSON)
     exit, status = commands.create(
         git_name,
         git_mail,
         identifier, [config.TEMPLATE_CJSON, config.TEMPLATE_EAD],
         agent=agent)
     return exit, status
Esempio n. 13
0
 def write_json(self, doc_metadata=True, obj_metadata={}, force=False, path=None):
     """Write Collection/Entity JSON file to disk.
     
     @param doc_metadata: boolean
     @param obj_metadata: dict Cached results of object_metadata.
     @param force: boolean Write even nothing looks changed.
     @param path: str Alternate absolute file path
     """
     if force or self.is_modified():
         if not path:
             path = self.json_path
         if not os.path.exists(os.path.dirname(path)):
             os.makedirs(os.path.dirname(path))
         fileio.write_text(
             self.dump_json(
                 doc_metadata=doc_metadata, obj_metadata=obj_metadata
             ),
             path
         )
Esempio n. 14
0
 def new(identifier, git_name, git_mail, agent='cmdln'):
     """Creates new Entity, writes to filesystem, does initial commit.
     
     @param identifier: Identifier
     @param git_name: str
     @param git_mail: str
     @param agent: str
     @returns: exit,status int,str
     """
     collection = identifier.collection().object()
     if not collection:
         raise Exception('Parent collection for %s does not exist.' % identifier)
     entity = Entity.create(identifier.path_abs(), identifier)
     fileio.write_text(
         entity.dump_json(template=True),
         config.TEMPLATE_EJSON
     )
     exit,status = commands.entity_create(
         git_name, git_mail,
         collection, entity.identifier,
         [collection.json_path_rel, collection.ead_path_rel],
         [config.TEMPLATE_EJSON, config.TEMPLATE_METS],
         agent=agent
     )
     if exit:
         raise Exception('Could not create new Entity: %s, %s' % (exit, status))
     # load Entity object, inherit values from parent, write back to file
     entity = Entity.from_identifier(identifier)
     entity.inherit(collection)
     entity.write_json()
     updated_files = [entity.json_path]
     exit,status = commands.entity_update(
         git_name, git_mail,
         collection, entity,
         updated_files,
         agent=agent
     )
     return exit,status
Esempio n. 15
0
 def new(identifier, git_name, git_mail, agent='cmdln'):
     """Creates new Entity, writes to filesystem, does initial commit.
     
     @param identifier: Identifier
     @param git_name: str
     @param git_mail: str
     @param agent: str
     @returns: exit,status int,str
     """
     collection = identifier.collection().object()
     if not collection:
         raise Exception('Parent collection for %s does not exist.' % identifier)
     entity = Entity.create(identifier)
     fileio.write_text(
         entity.dump_json(template=True),
         config.TEMPLATE_EJSON
     )
     exit,status = commands.entity_create(
         git_name, git_mail,
         collection, entity.identifier,
         [collection.json_path_rel, collection.ead_path_rel],
         [config.TEMPLATE_EJSON, config.TEMPLATE_METS],
         agent=agent
     )
     if exit:
         raise Exception('Could not create new Entity: %s, %s' % (exit, status))
     # load Entity object, inherit values from parent, write back to file
     entity = Entity.from_identifier(identifier)
     entity.inherit(collection)
     entity.write_json()
     updated_files = [entity.json_path]
     exit,status = commands.entity_update(
         git_name, git_mail,
         collection, entity,
         updated_files,
         agent=agent
     )
     return exit,status
Esempio n. 16
0
    def create(collection, entity_id, git_name, git_mail, agent=settings.AGENT):
        """create new entity given an entity ID
        TODO remove write and commit, just create object
        """
        eidentifier = Identifier(id=entity_id)
        entity_path = eidentifier.path_abs()
        
        # write entity.json template to entity location and commit
        fileio.write_text(Entity(entity_path).dump_json(template=True),
                   settings.TEMPLATE_EJSON)
        exit,status = commands.entity_create(
            git_name, git_mail,
            collection, eidentifier,
            [collection.json_path_rel, collection.ead_path_rel],
            [settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
            agent=agent)
        
        # load new entity, inherit values from parent, write and commit
        entity = Entity.from_json(entity_path)
        entity.inherit(collection)
        entity.write_json()
        updated_files = [entity.json_path]
        exit,status = commands.entity_update(
            git_name, git_mail,
            collection, entity,
            updated_files,
            agent=agent)

        # delete cache, update search index
        collection.cache_delete()
        with open(entity.json_path, 'r') as f:
            document = json.loads(f.read())
        try:
            docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document)
        except ConnectionError:
            logger.error('Could not post to Elasticsearch.')
        
        return entity
Esempio n. 17
0
 def gitignore(self):
     if not os.path.exists(self.gitignore_path):
         fileio.write_text(fileio.read_text(GITIGNORE_TEMPLATE),
                           self.gitignore_path)
     return fileio.read_text(self.gitignore_path)
Esempio n. 18
0
def new( request, repo, org, cid ):
    """Gets new EID from workbench, creates new entity record.
    
    If it messes up, goes back to collection.
    """
    git_name = request.session.get('git_name')
    git_mail = request.session.get('git_mail')
    if not (git_name and git_mail):
        messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
    collection = Collection.from_request(request)
    if collection.locked():
        messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
        return HttpResponseRedirect(collection.absolute_url())
    collection.repo_fetch()
    if collection.repo_behind():
        messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
        return HttpResponseRedirect(collection.absolute_url())
    
    ic = idservice.IDServiceClient()
    # resume session
    auth_status,auth_reason = ic.resume(request.session['idservice_token'])
    if auth_status != 200:
        request.session['idservice_username'] = None
        request.session['idservice_token'] = None
        messages.warning(
            request,
            'Session resume failed: %s %s (%s)' % (
                auth_status,auth_reason,settings.IDSERVICE_API_BASE
            )
        )
        return HttpResponseRedirect(collection.absolute_url())
    # get new entity ID
    http_status,http_reason,new_entity_id = ic.next_object_id(
        collection.identifier,
        'entity'
    )
    if http_status not in [200,201]:
        err = '%s %s' % (http_status, http_reason)
        msg = WEBUI_MESSAGES['VIEWS_ENT_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err)
        logger.error(msg)
        messages.error(request, msg)
        return HttpResponseRedirect(collection.absolute_url())
    
    eidentifier = Identifier(id=new_entity_id)
    # create new entity
    entity_path = eidentifier.path_abs()
    # write entity.json template to entity location
    fileio.write_text(
        Entity(entity_path).dump_json(template=True),
        settings.TEMPLATE_EJSON
    )
    
    # commit files
    exit,status = commands.entity_create(
        git_name, git_mail,
        collection, eidentifier,
        [collection.json_path_rel, collection.ead_path_rel],
        [settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
        agent=settings.AGENT
    )
    
    # load Entity object, inherit values from parent, write back to file
    entity = Entity.from_identifier(eidentifier)
    entity.inherit(collection)
    entity.write_json()
    updated_files = [entity.json_path]
    exit,status = commands.entity_update(
        git_name, git_mail,
        collection, entity,
        updated_files,
        agent=settings.AGENT
    )
    
    collection.cache_delete()
    if exit:
        logger.error(exit)
        logger.error(status)
        messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
    else:
        # update search index
        try:
            entity.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
        except ConnectionError:
            logger.error('Could not post to Elasticsearch.')
        gitstatus_update.apply_async((collection.path,), countdown=2)
        # positive feedback
        return HttpResponseRedirect(reverse('webui-entity-edit', args=entity.idparts))
    
    # something happened...
    logger.error('Could not create new entity!')
    messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_ERR_CREATE'])
    return HttpResponseRedirect(collection.absolute_url())
Esempio n. 19
0
 def write_xml(self):
     """Write METS XML file to disk.
     """
     fileio.write_text(self.dump_xml(), self.mets_path)
Esempio n. 20
0
def import_entities(csv_path, collection_path, git_name, git_mail):
    """
    @param csv_path: Absolute path to CSV data file.
    @param collection_path: Absolute path to collection repo.
    @param git_name: Username for use in changelog, git log
    @param git_mail: User email address for use in changelog, git log
    """
    rows = read_csv(csv_path)
    headers = rows[0]
    rows = rows[1:]

    headers = replace_variant_headers("entity", headers)
    field_names = FIELD_NAMES["entity"]
    nonrequired_fields = REQUIRED_FIELDS_EXCEPTIONS["entity"]
    required_fields = get_required_fields(ENTITY_FIELDS, nonrequired_fields)
    rows = replace_variant_cv_field_values("entity", headers, rows)
    invalid_headers("entity", headers, field_names, nonrequired_fields)
    all_rows_valid("entity", headers, required_fields, rows)
    if True:
        collection = Collection.from_json(collection_path)
        print(collection)

        # --------------------------------------------------
        def prep_creators(data):
            return [x.strip() for x in data.strip().split(";") if x]

        def prep_language(data):
            """language can be 'eng', 'eng;jpn', 'eng:English', 'jpn:Japanese'
            """
            y = []
            for x in data.strip().split(";"):
                if ":" in x:
                    y.append(x.strip().split(":")[0])
                else:
                    y.append(x.strip())
            return y

        def prep_topics(data):
            return [x.strip() for x in data.strip().split(";") if x]

        def prep_persons(data):
            return [x.strip() for x in data.strip().split(";") if x]

        def prep_facility(data):
            return [x.strip() for x in data.strip().split(";") if x]

        # --------------------------------------------------

        print("Data file looks ok")
        started = datetime.now()
        print("%s starting import" % dtfmt(started))
        print("")
        for n, row in enumerate(rows):
            rowstarted = datetime.now()
            rowd = make_row_dict(headers, row)

            # create new entity
            entity_id = rowd["id"]
            entity_path = os.path.join(collection_path, COLLECTION_FILES_PREFIX, entity_id)

            # write entity.json template to entity location
            fileio.write_text(Entity(entity_path).dump_json(template=True), TEMPLATE_EJSON)
            # commit files
            exit, status = commands.entity_create(
                git_name,
                git_mail,
                collection.path,
                entity_id,
                [collection.json_path_rel, collection.ead_path_rel],
                [TEMPLATE_EJSON, TEMPLATE_METS],
                agent=AGENT,
            )

            # reload newly-created Entity object
            entity = Entity.from_json(entity_path)

            # preppers
            rowd["creators"] = prep_creators(rowd["creators"])
            rowd["language"] = prep_language(rowd["language"])
            rowd["topics"] = prep_topics(rowd["topics"])
            rowd["persons"] = prep_persons(rowd["persons"])
            rowd["facility"] = prep_facility(rowd["facility"])

            # insert values from CSV
            for key in rowd.keys():
                setattr(entity, key, rowd[key])
            entity.record_created = datetime.now()
            entity.record_lastmod = datetime.now()

            # write back to file
            entity.write_json()
            updated_files = [entity.json_path]
            exit, status = commands.entity_update(
                git_name, git_mail, entity.parent_path, entity.id, updated_files, agent=AGENT
            )

            rowfinished = datetime.now()
            rowelapsed = rowfinished - rowstarted
            print("%s %s/%s %s (%s)" % (dtfmt(rowfinished), n + 1, len(rows), entity.id, rowelapsed))
        finished = datetime.now()
        elapsed = finished - started
        print("")
        print("%s done (%s rows)" % (dtfmt(finished), len(rows)))
        print("%s elapsed" % elapsed)
        print("")
Esempio n. 21
0
def new( request, repo, org ):
    """Gets new CID from workbench, creates new collection record.
    
    If it messes up, goes back to collection list.
    """
    oidentifier = Identifier(request)
    git_name = request.session.get('git_name')
    git_mail = request.session.get('git_mail')
    if not (git_name and git_mail):
        messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
    
    ic = idservice.IDServiceClient()
    # resume session
    auth_status,auth_reason = ic.resume(request.session['idservice_token'])
    if auth_status != 200:
        request.session['idservice_username'] = None
        request.session['idservice_token'] = None
        messages.warning(
            request,
            'Session resume failed: %s %s (%s)' % (
                auth_status,auth_reason,settings.IDSERVICE_API_BASE
            )
        )
        return HttpResponseRedirect(reverse('webui-collections'))
    # get new collection ID
    http_status,http_reason,collection_id = ic.next_object_id(
        oidentifier,
        'collection'
    )
    if http_status not in [200,201]:
        err = '%s %s' % (http_status, http_reason)
        msg = WEBUI_MESSAGES['VIEWS_COLL_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err)
        logger.error(msg)
        messages.error(request, msg)
        return HttpResponseRedirect(reverse('webui-collections'))
    
    identifier = Identifier(id=collection_id, base_path=settings.MEDIA_BASE)
    # create the new collection repo
    collection_path = identifier.path_abs()
    # collection.json template
    fileio.write_text(
        Collection(collection_path).dump_json(template=True),
        settings.TEMPLATE_CJSON
    )
    exit,status = commands.create(
        git_name, git_mail,
        identifier,
        [settings.TEMPLATE_CJSON, settings.TEMPLATE_EAD],
        agent=settings.AGENT
    )
    if exit:
        logger.error(exit)
        logger.error(status)
        messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
    else:
        # update search index
        collection = Collection.from_identifier(identifier)
        try:
            collection.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
        except ConnectionError:
            logger.error('Could not post to Elasticsearch.')
        gitstatus_update.apply_async((collection_path,), countdown=2)
        # positive feedback
        return HttpResponseRedirect( reverse('webui-collection-edit', args=collection.idparts) )
    # something happened...
    logger.error('Could not create new collecion!')
    messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_ERR_CREATE'])
    return HttpResponseRedirect(reverse('webui-collections'))
Esempio n. 22
0
 def create(path: str, collection_id: str, entity_id: str):
     logging.debug('    EntityControlFile.create({})'.format(path))
     t = load_template(ENTITY_CONTROL_TEMPLATE)
     fileio.write_text(t.format(cid=collection_id, eid=entity_id), path)
Esempio n. 23
0
 def create(path: str, collection_id: str):
     logging.debug('    CollectionControlFile.create({})'.format(path))
     t = load_template(COLLECTION_CONTROL_TEMPLATE)
     fileio.write_text(t.format(cid=collection_id), path)
Esempio n. 24
0
 def write_xml(self):
     """Write EAD XML file to disk.
     """
     fileio.write_text(self.dump_xml(), self.ead_path)
Esempio n. 25
0
 def create( path, collection_id ):
     logging.debug('    CollectionControlFile.create({})'.format(path))
     t = load_template(COLLECTION_CONTROL_TEMPLATE)
     fileio.write_text(t.format(cid=collection_id), path)
Esempio n. 26
0
 def write_xml(self):
     """Write METS XML file to disk.
     """
     fileio.write_text(self.dump_xml(), self.mets_path)