def parse_errands(self, ip, rootdir, archive, errands_root):
archive_structure = archive.get_active_structure()
structure = archive_structure.structure
for errand in self.get_arkiv_objekt_arenden(errands_root):
component, structure_unit = self.parse_errand(errand, archive, ip, structure)
tag = Tag(information_package=ip, task=self.task)
tag_version = TagVersion(pk=component.meta.id, tag=tag,
elastic_index=component._index._name,
name=component.name, type=component.type,
reference_code=component.reference_code)
tag_repr = TagStructure(
tag=tag,
structure_unit=structure_unit,
structure=structure,
parent=archive_structure,
tree_id=archive_structure.tree_id,
lft=0,
rght=0,
level=0,
)
acts_root = self.get_acts_root(errand)
if len(acts_root):
for act in self.parse_acts(ip, rootdir, component, acts_root[0], tag_repr):
yield act
yield tag, tag_version, tag_repr, component.to_dict(include_meta=True)
def parse_document(self, ip, rootdir, document, act, parent):
id = str(uuid.uuid4())
name = document.get("Namn")
desc = document.get("Beskrivning")
filepath = document.get('Lank')
if ip is not None:
filepath = os.path.join(ip.object_path, ip.sip_path, document.get('Lank'))
elif rootdir is not None:
filepath = os.path.join(rootdir, document.get('Lank'))
href = os.path.dirname(os.path.relpath(filepath, rootdir))
href = '' if href == '.' else href
filename = os.path.basename(filepath)
ext = os.path.splitext(filepath)[1][1:]
with open(filepath, 'rb') as f:
content = f.read()
encoded_content = base64.b64encode(content).decode("ascii")
size, _ = get_tree_size_and_count(filepath)
modified = timestamp_to_datetime(os.stat(filepath).st_mtime)
d = File(
_id=id,
name=name,
type='Bilaga',
archive=act.archive,
desc=desc,
filename=filename,
href=href,
extension=ext,
data=encoded_content,
size=size,
modified=modified,
current_version=True,
ip=act.ip,
task_id=str(self.task.pk),
)
tag = Tag(information_package=ip, task=self.task)
tag_version = TagVersion(pk=d.meta.id, tag=tag,
elastic_index=d._index._name,
name=d.name, type=d.type,
reference_code='')
tag_repr = TagStructure(
tag=tag,
parent=parent,
structure=parent.structure,
tree_id=parent.tree_id,
lft=0,
rght=0,
level=0,
)
self.indexed_files.append(filepath)
d_dict = d.to_dict(include_meta=True)
d_dict['pipeline'] = 'ingest_attachment'
return tag, tag_version, tag_repr, d_dict
def parse_volym(cls, el, archive_version, parent_tag_structure, structure_unit, agent, task=None, ip=None):
logger.debug("Parsing volym...")
ref_code = el.xpath("va:volnr", namespaces=cls.NSMAP)[0].text
name = el.xpath("va:utseende", namespaces=cls.NSMAP)[0].text
tag_type = cls.VOLUME_TYPE
volym_id = uuid.uuid4()
tag = Tag(information_package=ip, task=task)
tag_version = TagVersion(
pk=volym_id,
tag=tag,
elastic_index='component',
reference_code=ref_code,
name=name,
create_date=cls.parse_volume_create_date(el),
revise_date=cls.parse_volume_revise_date(el),
import_date=timezone.now(),
type=tag_type,
)
tag_structure = TagStructure(
tag=tag,
structure_unit=structure_unit,
structure=parent_tag_structure.structure,
parent=parent_tag_structure,
tree_id=parent_tag_structure.tree_id,
lft=0,
rght=0,
level=0
)
agent_tag_link = AgentTagLink(
agent=agent,
tag_id=tag_version.id,
type=cls.AGENT_TAG_LINK_RELATION_TYPE,
)
doc = Component.from_obj(tag_version, archive=archive_version)
doc.agents = [str(agent.pk)]
logger.debug("Parsed volym: {}".format(tag_version.pk))
return doc.to_dict(include_meta=True), tag, tag_version, tag_structure, agent_tag_link
def parse_acts(self, ip, rootdir, errand, acts_root, parent):
for act_el in acts_root.xpath("*[local-name()='ArkivobjektHandling']"):
act = self.parse_act(act_el, errand)
tag = Tag(information_package=ip, task=self.task)
tag_version = TagVersion(pk=act.meta.id, tag=tag,
elastic_index=act._index._name,
name=act.name, type=act.type,
reference_code=act.reference_code)
tag_repr = TagStructure(
tag=tag,
parent=parent,
structure=parent.structure,
tree_id=parent.tree_id,
lft=0,
rght=0,
level=0
)
for doc_el in act_el.xpath("*[local-name()='Bilaga']"):
yield self.parse_document(ip, rootdir, doc_el, act, tag_repr)
yield tag, tag_version, tag_repr, act.to_dict(include_meta=True)
def create(self, validated_data):
with transaction.atomic():
structure_unit = validated_data.pop('structure_unit', None)
parent = validated_data.pop('parent', None)
structure = validated_data.pop('structure', None)
notes_data = validated_data.pop('notes', [])
identifiers_data = validated_data.pop('identifiers', [])
information_package = validated_data.pop('information_package', None)
appraisal_date = validated_data.pop('appraisal_date', None)
index = validated_data.pop('index')
tag = Tag.objects.create(
information_package=information_package,
appraisal_date=appraisal_date,
)
tag_structure = TagStructure(tag=tag)
if structure_unit is not None:
tag_structure.structure_unit = structure_unit
tag_structure.structure = structure_unit.structure
archive_structure = TagStructure.objects.filter(structure=structure_unit.structure).first().get_root()
tag_structure.parent = archive_structure
tag_structure.save()
else:
if structure is None:
parent_structure = parent.get_active_structure()
else:
parent_structure = parent.get_structures(structure).get()
tag_structure.parent = parent_structure
tag_structure.structure = parent_structure.structure
tag_structure.save()
tag_version = TagVersion.objects.create(
tag=tag, elastic_index=index, **validated_data,
)
tag.current_version = tag_version
tag.save()
for agent_link in AgentTagLink.objects.filter(tag=tag_version):
AgentTagLink.objects.create(tag=tag_version, agent=agent_link.agent, type=agent_link.type)
tag_structure.refresh_from_db()
# Prel remove code, does not need copy to related structures??
# if structure_unit is None:
# structure_unit = tag_structure.get_ancestors(
# include_self=True
# ).filter(structure_unit__isnull=False).get().structure_unit
# related_units = structure_unit.related_structure_units.filter(
# structure__is_template=False
# ).exclude(
# structure=tag_structure.structure
# )
# logger.debug('ComponentWriteSerializer - create - tag: {tag}, tag_structure: {tag_structure},
# tag_structure.structure: {tag_structure_structure},
# tag_structure.structure_unit: {tag_structure_structure_unit},
# related_units: {related_units}'.format(
# tag=tag, tag_structure=tag_structure, tag_structure_structure=tag_structure.structure,
# tag_structure_structure_unit=tag_structure.structure_unit, related_units=related_units
# ))
# for related in related_units:
# new_unit = related if tag_structure.structure_unit is not None else None
# logger.debug('ComponentWriteSerializer - create (for related) - related: {related},
# new_unit: {new_unit}, related.structure: {related_structure}'.format(
# related=related, new_unit=new_unit, related_structure=related.structure
# ))
# new_tag_structure = tag_structure.copy_to_new_structure(related.structure, new_unit=new_unit)
# logger.debug('ComponentWriteSerializer - create (for related 2) - new_tag_structure:
# {new_tag_structure}'.format(
# new_tag_structure=new_tag_structure
# ))
self.create_identifiers(tag_version, identifiers_data)
self.create_notes(tag_version, notes_data)
tag_version.refresh_from_db()
doc = Component.from_obj(tag_version)
doc.save()
return tag
def create(self, request):
serializer = SearchSerializer(data=request.data)
organization = request.user.user_profile.current_organization
if serializer.is_valid(raise_exception=True):
data = serializer.validated_data
if data.get('index') == 'archive':
if not request.user.has_perm('tags.create_archive'):
raise exceptions.PermissionDenied(
'You do not have permission to create new archives')
if organization is None:
raise exceptions.ParseError(
'You must be part of an organization to create a new archive'
)
else:
if not request.user.has_perm('tags.add_tag'):
raise exceptions.PermissionDenied(
'You do not have permission to create nodes')
with transaction.atomic():
tag = Tag.objects.create()
tag_structure = TagStructure(tag=tag)
structure = data.pop('structure')
if data.get('index') != 'archive':
if 'structure_unit' in data:
structure_unit = StructureUnit.objects.get(
pk=data.pop('structure_unit'))
tag_structure.structure_unit = structure_unit
tag_structure.structure = structure_unit.structure
archive_structure = TagStructure.objects.get(
tag__versions=data.pop('archive'))
tag_structure.parent = archive_structure
tag_structure.save()
elif 'parent' in data:
parent_version = TagVersion.objects.select_for_update(
).get(pk=data.pop('parent'))
if structure is None:
parent_structure = parent_version.get_active_structure(
)
else:
parent_structure = parent_version.get_structures(
structure).get()
tag_structure.parent = parent_structure
tag_structure.structure = parent_structure.structure
elif structure is not None:
tag_structure.structure = structure
tag_structure.save()
tag_version = TagVersion.objects.create(
tag=tag,
elastic_index=data['index'],
name=data['name'],
type=data['type'],
reference_code=data['reference_code'])
tag.current_version = tag_version
tag.save()
self._update_tag_metadata(tag_version, data)
if tag_version.elastic_index == 'archive':
org = request.user.user_profile.current_organization
org.add_object(tag_version)
search_data = {}
if data.get('archive_creator'):
search_data['archive_creator'] = data.get(
'archive_creator')
if data.get('archive_responsible'):
search_data['archive_responsible'] = data.get(
'archive_responsible')
if search_data:
self._update_tag_metadata(tag_version, search_data)
# create descendants from structure
for unit in tag_structure.structure.units.all():
tag = Tag.objects.create()
tv = TagVersion.objects.create(
tag=tag,
elastic_index='component',
name=unit.name,
reference_code=unit.reference_code,
type=unit.type)
tv.update_search({
'archive': str(tag_version.pk),
'desc': unit.description
})
tag.save()
ts = TagStructure(tag=tag, structure=structure)
if unit.parent is not None:
parent = unit.parent.reference_code
ts.parent = TagStructure.objects.filter(tree_id=tag_structure.tree_id,
tag__versions__elastic_index='component')\
.filter(tag__versions__reference_code=parent)\
.distinct().get()
else:
ts.parent = tag_structure
ts.save()
return Response(self.serialize(tag_version.to_search()),
status=status.HTTP_201_CREATED)
def create(self, request):
serializer = SearchSerializer(data=request.data)
organization = request.user.user_profile.current_organization
if serializer.is_valid(raise_exception=True):
data = serializer.validated_data
if data.get('index') == 'archive':
if not request.user.has_perm('tags.create_archive'):
raise exceptions.PermissionDenied('You do not have permission to create new archives')
if organization is None:
raise exceptions.ParseError('You must be part of an organization to create a new archive')
else:
if not request.user.has_perm('tags.add_tag'):
raise exceptions.PermissionDenied('You do not have permission to create nodes')
with transaction.atomic():
tag = Tag.objects.create()
tag_structure = TagStructure(tag=tag)
structure = data.pop('structure')
if data.get('index') != 'archive':
if 'structure_unit' in data:
structure_unit = StructureUnit.objects.get(pk=data.pop('structure_unit'))
tag_structure.structure_unit = structure_unit
tag_structure.structure = structure_unit.structure
archive_structure = TagStructure.objects.get(tag__versions=data.pop('archive'))
tag_structure.parent = archive_structure
tag_structure.save()
elif 'parent' in data:
parent_version = TagVersion.objects.select_for_update().get(pk=data.pop('parent'))
if structure is None:
parent_structure = parent_version.get_active_structure()
else:
parent_structure = parent_version.get_structures(structure).get()
tag_structure.parent = parent_structure
tag_structure.structure = parent_structure.structure
elif structure is not None:
tag_structure.structure = structure
tag_structure.save()
tag_version = TagVersion.objects.create(
tag=tag, elastic_index=data['index'], name=data['name'],
type=data['type'], reference_code=data['reference_code'])
tag.current_version = tag_version
tag.save()
self._update_tag_metadata(tag_version, data)
if tag_version.elastic_index == 'archive':
org = request.user.user_profile.current_organization
org.add_object(tag_version)
search_data = {}
if data.get('archive_creator'):
search_data['archive_creator'] = data.get('archive_creator')
if data.get('archive_responsible'):
search_data['archive_responsible'] = data.get('archive_responsible')
if search_data:
self._update_tag_metadata(tag_version, search_data)
# create descendants from structure
for unit in tag_structure.structure.units.all():
tag = Tag.objects.create()
tv = TagVersion.objects.create(tag=tag, elastic_index='component', name=unit.name,
reference_code=unit.reference_code, type=unit.type)
tv.update_search({'archive': str(tag_version.pk), 'desc': unit.description})
tag.save()
ts = TagStructure(tag=tag, structure=structure)
if unit.parent is not None:
parent = unit.parent.reference_code
ts.parent = TagStructure.objects.filter(tree_id=tag_structure.tree_id,
tag__versions__elastic_index='component')\
.filter(tag__versions__reference_code=parent)\
.distinct().get()
else:
ts.parent = tag_structure
ts.save()
return Response(self.serialize(tag_version.to_search()), status=status.HTTP_201_CREATED)
def create(self, validated_data):
with transaction.atomic():
structure_unit = validated_data.pop('structure_unit', None)
parent = validated_data.pop('parent', None)
structure = validated_data.pop('structure', None)
notes_data = validated_data.pop('notes', [])
identifiers_data = validated_data.pop('identifiers', [])
information_package = validated_data.pop('information_package', None)
index = validated_data.pop('index')
tag = Tag.objects.create(information_package=information_package)
tag_structure = TagStructure(tag=tag)
if structure_unit is not None:
tag_structure.structure_unit = structure_unit
tag_structure.structure = structure_unit.structure
archive_structure = TagStructure.objects.filter(structure=structure_unit.structure).first().get_root()
tag_structure.parent = archive_structure
tag_structure.save()
else:
if structure is None:
parent_structure = parent.get_active_structure()
else:
parent_structure = parent.get_structures(structure).get()
tag_structure.parent = parent_structure
tag_structure.structure = parent_structure.structure
tag_structure.save()
tag_version = TagVersion.objects.create(
tag=tag, elastic_index=index, **validated_data,
)
tag.current_version = tag_version
tag.save()
for agent_link in AgentTagLink.objects.filter(tag=tag_version):
AgentTagLink.objects.create(tag=tag_version, agent=agent_link.agent, type=agent_link.type)
tag_structure.refresh_from_db()
if structure_unit is None:
structure_unit = tag_structure.get_ancestors(
include_self=True
).filter(structure_unit__isnull=False).get().structure_unit
related_units = structure_unit.related_structure_units.filter(
structure__is_template=False
).exclude(
structure=tag_structure.structure
)
for related in related_units:
new_unit = related if tag_structure.structure_unit is not None else None
tag_structure.copy_to_new_structure(related.structure, new_unit=new_unit)
self.create_identifiers(self, identifiers_data)
self.create_notes(self, notes_data)
doc = Component.from_obj(tag_version)
doc.save()
return tag