def handle(self, *args, **options): FedoraTypeManager.populate() models = list(args) models.sort() fields = {} model_classes = set() for model_name in models: model_name = model_name.split('.') module_name = '.'.join(model_name[:-1]) model_name = model_name[-1] class_for_name(module_name, model_name) modelclz = FedoraTypeManager.get_model_class(model_name) for clz in inspect.getmro(modelclz): if clz == FedoraObject or issubclass(clz, FedoraObject): model_classes.add(clz.__name__) for field in modelclz.indexed_fields: if field.name not in fields: fields[field.name] = field else: fields[field.name] = field.combine(fields[field.name]) print(field) print() ldpath = ['id = . :: xsd:string ;'] all_fields = [] for field in fields.values(): field_indexer_name = indexer_name(field) field_rdf_name = field.rdf_name if field_indexer_name.endswith('_t'): if MULTI_LANG.issubset(field.field_type): from django.conf import settings langstrings = [] for langi, lang in enumerate(settings.LANGUAGES): if langi: langstrings.append(', " ", ') langstrings.append('<{0}>[@{1}]'.format( field_rdf_name, lang[0])) all_fields.append('<{0}>[@{1}]'.format( field_rdf_name, lang[0])) all_fields.append('" "') ldpath.append(""" {0} = fn:concat({1}) :: {2}; """.format(field_indexer_name, ''.join(langstrings), field.xml_schema_type).strip()) else: ldpath.append(""" {0} = <{1}> :: {2}; """.format(field_indexer_name, field_rdf_name, field.xml_schema_type).strip()) all_fields.append('<%s>' % field_rdf_name) all_fields.append('" "') if MULTI_LANG.issubset(field.field_type): # multi lang field, generate all languages from settings.py from django.conf import settings for lang in settings.LANGUAGES: ldpath.append(""" {3}__{0} = <{1}>[@{3}] :: {2}; """.format(field_indexer_name, field_rdf_name, field.xml_schema_type, lang[0]).strip()) ldpath.append(""" sort__{3}__{0}_s = <{1}>[@{3}] :: {2}; """.format(field_indexer_name[:-3], field_rdf_name, field.xml_schema_type, lang[0]).strip()) else: ldpath.append(""" sort__{0}_s = <{1}> :: {2}; """.format(field_indexer_name[:-3], field_rdf_name, field.xml_schema_type).strip()) else: if MULTI_LANG.issubset(field.field_type): from django.conf import settings langstrings = [] for langi, lang in enumerate(settings.LANGUAGES): if langi: langstrings.append(', " ", ') langstrings.append('<{0}>[@{1}]'.format( field_rdf_name, lang[0])) all_fields.append('<{0}>[@{1}]'.format( field_rdf_name, lang[0])) all_fields.append('" "') ldpath.append(""" {0} = fn:concat({1}) :: {2}; """.format(field_indexer_name, ''.join(langstrings), field.xml_schema_type).strip()) else: ldpath.append(""" {0} = <{1}> :: {2}; """.format(field_indexer_name, field_rdf_name, field.xml_schema_type).strip()) if MULTI_LANG.issubset(field.field_type): # multi lang field, generate all languages from settings.py from django.conf import settings for lang in settings.LANGUAGES: ldpath.append(""" {3}__{0} = <{1}>[@{3}] :: {2}; """.format(field_indexer_name, field_rdf_name, field.xml_schema_type, lang[0]).strip()) ldpath.append( 'fedora_parent_id_t = <http://fedora.info/definitions/v4/repository#hasParent> :: xsd:string;' ) ldpath.append('fedora_mixin_types_t = "%s" :: xsd:string;' % ' '.join(model_classes)) if all_fields: ldpath.append('solr_all_fields_t = fn:concat(%s) :: xsd:string;' % ', '.join(all_fields)) ldpath = '\n'.join(ldpath) print(ldpath) name = '__'.join( [inflection.underscore(x.split('.')[-1]) for x in models]) mgr = FedoraManager.get_manager() mgr.connection.direct_put( 'fedora:system/fedora:transform/fedora:ldpath/' + name + "/fedora:Container", data=ldpath, content_type='application/rdf+ldpath') # prepare schema inside solr - not needed, using wildcards # solr = FedoraManager.get_indexer() # for field in fields.values(): # solr.install_indexed_field(field) log.info("Transformation created with name `%s'", name)
def handle(self, *args, **options): FedoraTypeManager.populate() models = list(args) models.sort() fields = {} model_classes = set() for model_name in models: model_name = model_name.split('.') module_name = '.'.join(model_name[:-1]) model_name = model_name[-1] class_for_name(module_name, model_name) modelclz = FedoraTypeManager.get_model_class(model_name) for clz in inspect.getmro(modelclz): if clz == FedoraObject or issubclass(clz, FedoraObject): model_classes.add(clz.__name__) for field in modelclz.indexed_fields: if field.name not in fields: fields[field.name] = field else: fields[field.name] = field.combine(fields[field.name]) print(field) print() ldpath = [ 'id = . :: xsd:string ;' ] all_fields = [] for field in fields.values(): field_indexer_name = indexer_name(field) field_rdf_name = field.rdf_name if field_indexer_name.endswith('_t'): if MULTI_LANG.issubset(field.field_type): from django.conf import settings langstrings = [] for langi, lang in enumerate(settings.LANGUAGES): if langi: langstrings.append(', " ", ') langstrings.append('<{0}>[@{1}]'.format(field_rdf_name, lang[0])) all_fields.append('<{0}>[@{1}]'.format(field_rdf_name, lang[0])) all_fields.append('" "') ldpath.append( """ {0} = fn:concat({1}) :: {2}; """.format(field_indexer_name, ''.join(langstrings), field.xml_schema_type).strip() ) else: ldpath.append( """ {0} = <{1}> :: {2}; """.format(field_indexer_name, field_rdf_name, field.xml_schema_type).strip() ) all_fields.append('<%s>' % field_rdf_name) all_fields.append('" "') if MULTI_LANG.issubset(field.field_type): # multi lang field, generate all languages from settings.py from django.conf import settings for lang in settings.LANGUAGES: ldpath.append( """ {3}__{0} = <{1}>[@{3}] :: {2}; """.format(field_indexer_name, field_rdf_name, field.xml_schema_type, lang[0]).strip() ) ldpath.append( """ sort__{3}__{0}_s = <{1}>[@{3}] :: {2}; """.format(field_indexer_name[:-3], field_rdf_name, field.xml_schema_type, lang[0]).strip() ) else: ldpath.append( """ sort__{0}_s = <{1}> :: {2}; """.format(field_indexer_name[:-3], field_rdf_name, field.xml_schema_type).strip() ) else: if MULTI_LANG.issubset(field.field_type): from django.conf import settings langstrings = [] for langi, lang in enumerate(settings.LANGUAGES): if langi: langstrings.append(', " ", ') langstrings.append('<{0}>[@{1}]'.format(field_rdf_name, lang[0])) all_fields.append('<{0}>[@{1}]'.format(field_rdf_name, lang[0])) all_fields.append('" "') ldpath.append( """ {0} = fn:concat({1}) :: {2}; """.format(field_indexer_name, ''.join(langstrings), field.xml_schema_type).strip() ) else: ldpath.append( """ {0} = <{1}> :: {2}; """.format(field_indexer_name, field_rdf_name, field.xml_schema_type).strip() ) if MULTI_LANG.issubset(field.field_type): # multi lang field, generate all languages from settings.py from django.conf import settings for lang in settings.LANGUAGES: ldpath.append( """ {3}__{0} = <{1}>[@{3}] :: {2}; """.format(field_indexer_name, field_rdf_name, field.xml_schema_type, lang[0]).strip() ) ldpath.append('fedora_parent_id_t = <http://fedora.info/definitions/v4/repository#hasParent> :: xsd:string;') ldpath.append('fedora_mixin_types_t = "%s" :: xsd:string;' % ' '.join(model_classes)) if all_fields: ldpath.append('solr_all_fields_t = fn:concat(%s) :: xsd:string;' % ', '.join(all_fields)) ldpath = '\n'.join(ldpath) print(ldpath) name = '__'.join([inflection.underscore(x.split('.')[-1]) for x in models]) mgr = FedoraManager.get_manager() mgr.connection.direct_put('fedora:system/fedora:transform/fedora:ldpath/' + name + "/fedora:Container", data=ldpath, content_type='application/rdf+ldpath') # prepare schema inside solr - not needed, using wildcards # solr = FedoraManager.get_indexer() # for field in fields.values(): # solr.install_indexed_field(field) log.info("Transformation created with name `%s'", name)
def handle(self, *args, **options): FedoraTypeManager.populate() models = list(args) for model_name in models: fields = {} split_model_name = model_name.split('.') indexer_model_name = '_'.join(split_model_name) module_name = '.'.join(split_model_name[:-1]) split_model_name = split_model_name[-1] class_for_name(module_name, split_model_name) modelclz = FedoraTypeManager.get_model_class(split_model_name) for field in modelclz._meta.fields: fldname = url2id(field.rdf_name) if fldname not in fields: fields[fldname] = field indexer = connections['repository'].indexer existing_mapping = indexer.get_mapping(indexer_model_name) existing_properties = existing_mapping.get('properties', {}) new_mapping = { } new_properties = { } fields['_fedora_id'] = FEDORA_ID_FIELD fields['_fedora_parent'] = FEDORA_PARENT_FIELD fields['_fedora_type'] = FEDORA_TYPE_FIELD fields['_fedoralink_model'] = FEDORALINK_TYPE_FIELD fields['_fedora_created'] = FEDORA_CREATED_FIELD fields['_fedora_last_modified'] = FEDORA_LAST_MODIFIED_FIELD for fldname, field in fields.items(): if fldname in existing_properties: continue props = {} new_properties[fldname] = props if isinstance(field, IndexedLanguageField): props['type'] = 'nested' props["include_in_root"] = 'true' props['properties'] = self.gen_languages_mapping(fldname + ".") elif isinstance(field, IndexedTextField): props['type'] = 'string' props['index'] = 'not_analyzed' props['copy_to'] = fldname + "__fulltext" new_properties[fldname + "__fulltext"] = { 'type': 'string', } elif isinstance(field, IndexedDateTimeField): props['type'] = 'date' props['index'] = 'not_analyzed' elif isinstance(field, IndexedDateField): props['type'] = 'date' props['index'] = 'not_analyzed' elif isinstance(field, IndexedIntegerField): props['type'] = 'long' props['index'] = 'not_analyzed' elif isinstance(field, IndexedGPSField): props['type'] = 'string' props['index'] = 'not_analyzed' elif isinstance(field, IndexedLinkedField) or isinstance(field, IndexedBinaryField) : props['type'] = 'string' props['index'] = 'not_analyzed' else: raise Exception("Mapping type %s not handled yet" % type(field)) new_mapping['_all'] = { "store": True } new_mapping['properties'] = new_properties print(json.dumps(new_mapping, indent=4)) indexer.save_mapping(indexer_model_name, new_mapping)
def handle(self, *args, **options): FedoraTypeManager.populate() models = options['model_name'] for model_name in models: fields = {} split_model_name = model_name.split('.') indexer_model_name = '_'.join(split_model_name) module_name = '.'.join(split_model_name[:-1]) split_model_name = split_model_name[-1] class_for_name(module_name, split_model_name) modelclz = FedoraTypeManager.get_model_class(split_model_name) for field in modelclz._meta.fields: fldname = url2id(field.rdf_name) if fldname not in fields: fields[fldname] = field indexer = connections['repository'].indexer existing_mapping = indexer.get_mapping(indexer_model_name) existing_properties = existing_mapping.get('properties', {}) new_mapping = {} new_properties = {} fields['_fedora_id'] = FEDORA_ID_FIELD fields['_fedora_parent'] = FEDORA_PARENT_FIELD fields['_fedora_type'] = FEDORA_TYPE_FIELD fields['_fedoralink_model'] = FEDORALINK_TYPE_FIELD fields['_fedora_created'] = FEDORA_CREATED_FIELD fields['_fedora_last_modified'] = FEDORA_LAST_MODIFIED_FIELD fields['_collection_child_types'] = CESNET_RDF_TYPES print('ADD fields to mapping') for fldname, field in fields.items(): if fldname in existing_properties: continue props = {} new_properties[fldname] = props if isinstance(field, IndexedLanguageField): props['type'] = 'nested' props["include_in_root"] = 'true' props['properties'] = self.gen_languages_mapping(fldname + ".") elif isinstance(field, IndexedTextField): props['type'] = 'keyword' props['copy_to'] = fldname + "__fulltext" new_properties[fldname + "__fulltext"] = { 'type': 'text', } elif isinstance(field, IndexedDateTimeField): props['type'] = 'date' elif isinstance(field, IndexedDateField): props['type'] = 'date' elif isinstance(field, IndexedIntegerField): props['type'] = 'long' elif isinstance(field, IndexedGPSField): props['type'] = 'keyword' elif isinstance(field, IndexedLinkedField) or isinstance( field, IndexedBinaryField): props['type'] = 'keyword' else: raise Exception("Mapping type %s not handled yet" % type(field)) new_mapping['_all'] = {"store": True} new_mapping['properties'] = new_properties print(json.dumps(new_mapping, indent=4)) indexer.save_mapping(indexer_model_name, new_mapping)