def submit(self, request, pk=None):
"""
Submits the specified information package
Args:
pk: The primary key (id) of the information package to submit
Returns:
None
"""
ip = self.get_object()
if ip.State != "Created":
raise ValueError(
"The IP (%s) is in the state '%s' but should be 'Created'" %
(pk, ip.State))
validators = request.data.get('validators', {})
validate_xml_file = validators.get('validate_xml_file', False)
validate_file_format = validators.get('validate_file_format', False)
validate_integrity = validators.get('validate_integrity', False)
validate_logical_physical_representation = validators.get(
'validate_logical_physical_representation', False)
step = ProcessStep.objects.create(name="Submit SIP",
information_package=ip)
step.tasks.add(
ProcessTask.objects.create(
name="preingest.tasks.UpdateIPStatus",
params={
"ip": ip,
"status": "Submitting",
},
processstep_pos=0,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
reception = Path.objects.get(entity="path_preingest_reception").value
sd_profile = ip.get_profile('submit_description')
container_format = ip.get_container_format()
container_file = os.path.join(
reception,
str(ip.pk) + ".%s" % container_format.lower())
sa = ip.SubmissionAgreement
info = sd_profile.fill_specification_data(sa, ip)
info["_IP_CREATEDATE"] = timestamp_to_datetime(
creation_date(container_file)).isoformat()
infoxml = os.path.join(reception, str(ip.pk) + ".xml")
filesToCreate = {infoxml: sd_profile.specification}
step.tasks.add(
ProcessTask.objects.create(
name="preingest.tasks.GenerateXML",
params={
"info": info,
"filesToCreate": filesToCreate,
"folderToParse": container_file,
"algorithm": ip.get_checksum_algorithm(),
},
processstep_pos=10,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
if validate_xml_file:
step.tasks.add(
ProcessTask.objects.create(
name="preingest.tasks.ValidateXMLFile",
params={"xml_filename": infoxml},
processstep_pos=14,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
if validate_file_format or validate_integrity:
step.tasks.add(
ProcessTask.objects.create(
name="preingest.tasks.ValidateFiles",
params={
"ip": ip,
"rootdir": reception,
"xmlfile": infoxml,
"validate_fileformat": validate_file_format,
"validate_integrity": validate_integrity,
},
processstep_pos=15,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
if validate_logical_physical_representation:
step.tasks.add(
ProcessTask.objects.create(
name=
"preingest.tasks.ValidateLogicalPhysicalRepresentation",
params={
"files": [os.path.basename(ip.ObjectPath)],
"xmlfile": infoxml,
},
processstep_pos=16,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
step.tasks.add(
ProcessTask.objects.create(
name="preingest.tasks.SubmitSIP",
params={"ip": ip},
processstep_pos=20,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
if ip.get_email_recipient():
recipients = [ip.get_email_recipient()]
subject = request.data.get('subject')
body = request.data.get('body')
attachments = [ip.ObjectPath]
step.tasks.add(
ProcessTask.objects.create(name="ESSArch_Core.tasks.SendEmail",
params={
'sender':
self.request.user.email,
'recipients': recipients,
'subject': subject,
'body': body,
'attachments': attachments
},
processstep_pos=25,
information_package=ip,
responsible=self.request.user))
step.tasks.add(
ProcessTask.objects.create(
name="preingest.tasks.UpdateIPStatus",
params={
"ip": ip,
"status": "Submitted"
},
processstep_pos=30,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
step.save()
step.run()
return Response({'status': 'submitting ip'})
def run(self, purpose=None, delete_sip=False):
self.logger.debug('Receiving SIP')
aip = InformationPackage.objects.get(pk=self.ip)
algorithm = aip.get_checksum_algorithm()
container = aip.object_path
objid, container_type = os.path.splitext(os.path.basename(container))
container_type = container_type.lower()
xml = aip.package_mets_path
aip.package_mets_create_date = timestamp_to_datetime(
creation_date(xml)).isoformat()
aip.package_mets_size = os.path.getsize(xml)
aip.package_mets_digest_algorithm = MESSAGE_DIGEST_ALGORITHM_CHOICES_DICT[
algorithm.upper()]
aip.package_mets_digest = calculate_checksum(xml, algorithm=algorithm)
aip.generation = 0
aic = InformationPackage.objects.create(
package_type=InformationPackage.AIC,
responsible=aip.responsible,
label=aip.label,
start_date=aip.start_date,
end_date=aip.end_date)
old_sip_path = aip.object_path
aip.aic = aic
aip_dir = os.path.join(aip.policy.ingest_path.value, objid)
aip.object_path = aip_dir
try:
os.makedirs(aip_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
aip.save()
dst_path, dst_name = find_destination('sip',
aip.get_profile('aip').structure,
aip.object_path)
if dst_path is None:
dst_path, dst_name = find_destination(
'content',
aip.get_profile('aip').structure, aip.object_path)
dst_name, = self.parse_params(dst_name)
dst = os.path.join(dst_path, dst_name)
sip_profile = aip.submission_agreement.profile_sip
try:
shutil.rmtree(dst)
except FileNotFoundError:
pass
if aip.policy.receive_extract_sip:
temp = Path.objects.cached('entity', 'temp', 'value')
with tempfile.TemporaryDirectory(dir=temp) as tmpdir:
self.logger.debug('Extracting {} to {}'.format(
container, tmpdir))
if container_type == '.tar':
with tarfile.open(container) as tar:
root_member_name = tar.getnames()[0]
tar.extractall(tmpdir)
elif container_type == '.zip':
with zipfile.ZipFile(container) as zipf:
root_member_name = zipf.namelist()[0]
zipf.extractall(tmpdir)
else:
raise ValueError(
'Invalid container type: {}'.format(container))
dst = os.path.join(dst, '')
try:
os.makedirs(dst)
except OSError as e:
if e.errno != errno.EEXIST:
raise
tmpsrc = tmpdir
if len(os.listdir(tmpdir)) == 1 and os.listdir(
tmpdir)[0] == root_member_name:
new_tmpsrc = os.path.join(tmpdir, root_member_name)
if os.path.isdir(new_tmpsrc):
tmpsrc = new_tmpsrc
self.logger.debug('Moving content of {} to {}'.format(
tmpsrc, dst))
for f in os.listdir(tmpsrc):
shutil.move(os.path.join(tmpsrc, f), dst)
self.logger.debug('Deleting {}'.format(tmpdir))
aip.sip_path = os.path.relpath(dst, aip.object_path)
else:
self.logger.debug('Copying {} to {}'.format(container, dst))
shutil.copy2(container, dst)
aip.sip_path = os.path.relpath(
os.path.join(dst, os.path.basename(container)),
aip.object_path)
sip_mets_dir, sip_mets_file = find_destination('mets_file',
sip_profile.structure,
aip.sip_path)
if os.path.isfile(aip.sip_path):
sip_mets_data = parse_mets(
open_file(
os.path.join(aip.object_path, sip_mets_dir, sip_mets_file),
container=aip.sip_path,
container_prefix=aip.object_identifier_value,
))
else:
sip_mets_data = parse_mets(
open_file(
os.path.join(aip.object_path, sip_mets_dir,
sip_mets_file)))
# prefix all SIP data
sip_mets_data = {
f'SIP_{k.upper()}': v
for k, v in sip_mets_data.items()
}
aip_profile_rel_data = aip.get_profile_rel('aip').data
aip_profile_rel_data.data.update(sip_mets_data)
aip_profile_rel_data.save()
if delete_sip:
delete_path(old_sip_path)
delete_path(pathlib.Path(old_sip_path).with_suffix('.xml'))
self.logger.debug('sip_path set to {}'.format(aip.sip_path))
aip.save()
def ReceiveSIP(self, purpose=None, delete_sip=False):
logger = logging.getLogger('essarch.workflow.tasks.ReceiveSIP')
logger.debug('Receiving SIP')
ip = self.get_information_package()
algorithm = ip.get_checksum_algorithm()
container = ip.object_path
objid, container_type = os.path.splitext(os.path.basename(container))
container_type = container_type.lower()
xml = ip.package_mets_path
ip.package_mets_create_date = timestamp_to_datetime(
creation_date(xml)).isoformat()
ip.package_mets_size = os.path.getsize(xml)
ip.package_mets_digest_algorithm = MESSAGE_DIGEST_ALGORITHM_CHOICES_DICT[
algorithm.upper()]
ip.package_mets_digest = calculate_checksum(xml, algorithm=algorithm)
ip.object_path = os.path.join(ip.policy.ingest_path.value,
ip.object_identifier_value)
ip.save()
sip_dst_path, sip_dst_name = find_destination('sip', ip.get_structure(),
ip.object_path)
if sip_dst_path is None:
sip_dst_path, sip_dst_name = find_destination('content',
ip.get_structure(),
ip.object_path)
sip_dst_name, = self.parse_params(sip_dst_name)
sip_dst = os.path.join(sip_dst_path, sip_dst_name)
if ip.policy.receive_extract_sip:
# remove any existing directory from previous attempts
delete_path(sip_dst)
temp = Path.objects.get(entity='temp').value
with tempfile.TemporaryDirectory(dir=temp) as tmpdir:
logger.debug('Extracting {} to {}'.format(container, tmpdir))
if container_type == '.tar':
with tarfile.open(container) as tar:
root_member_name = tar.getnames()[0]
tar.extractall(tmpdir)
elif container_type == '.zip':
with zipfile.ZipFile(container) as zipf:
root_member_name = zipf.namelist()[0]
zipf.extractall(tmpdir)
else:
raise ValueError(
'Invalid container type: {}'.format(container))
sip_dst = os.path.join(sip_dst, '')
os.makedirs(sip_dst)
tmpsrc = tmpdir
if len(os.listdir(tmpdir)) == 1 and os.listdir(
tmpdir)[0] == root_member_name:
new_tmpsrc = os.path.join(tmpdir, root_member_name)
if os.path.isdir(new_tmpsrc):
tmpsrc = new_tmpsrc
logger.debug('Moving content of {} to {}'.format(tmpsrc, sip_dst))
for f in os.listdir(tmpsrc):
shutil.move(os.path.join(tmpsrc, f), sip_dst)
logger.debug('Deleting {}'.format(tmpdir))
else:
logger.debug('Copying {} to {}'.format(container, sip_dst))
shutil.copy2(container, sip_dst)
ip.sip_path = os.path.relpath(sip_dst, ip.object_path)
ip.save()
self.create_success_event("Received SIP")
return sip_dst
def run(self, filepath=None, mimetype=None, relpath=None, algorithm='SHA-256'):
if not relpath:
relpath = filepath
relpath = win_to_posix(relpath)
timestamp = creation_date(filepath)
createdate = timestamp_to_datetime(timestamp)
checksum_task = ProcessTask.objects.create(
name="preingest.tasks.CalculateChecksum",
params={
"filename": filepath,
"algorithm": algorithm
}
)
fileformat_task = ProcessTask.objects.create(
name="preingest.tasks.IdentifyFileFormat",
params={
"filename": filepath,
}
)
checksum_task.log = self.taskobj.log
checksum_task.information_package = self.taskobj.information_package
checksum_task.responsible = self.taskobj.responsible
fileformat_task.log = self.taskobj.log
fileformat_task.information_package = self.taskobj.information_package
fileformat_task.responsible = self.taskobj.responsible
if self.taskobj is not None and self.taskobj.processstep is not None:
checksum_task.processstep = self.taskobj.processstep
fileformat_task.processstep = self.taskobj.processstep
checksum_task.save()
fileformat_task.save()
checksum = checksum_task.run_eagerly()
self.set_progress(50, total=100)
fileformat = fileformat_task.run_eagerly()
fileinfo = {
'FName': os.path.basename(relpath),
'FChecksum': checksum,
'FID': str(uuid.uuid4()),
'daotype': "borndigital",
'href': relpath,
'FMimetype': mimetype,
'FCreated': createdate.isoformat(),
'FFormatName': fileformat,
'FSize': str(os.path.getsize(filepath)),
'FUse': 'Datafile',
'FChecksumType': algorithm,
'FLoctype': 'URL',
'FLinkType': 'simple',
'FChecksumLib': 'hashlib',
'FLocationType': 'URI',
'FIDType': 'UUID',
}
self.set_progress(100, total=100)
return fileinfo
def submit(self, request, pk=None):
"""
Submits the specified information package
Args:
pk: The primary key (id) of the information package to submit
Returns:
None
"""
ip = self.get_object()
if ip.State != "Created":
raise ValueError(
"The IP (%s) is in the state '%s' but should be 'Created'" % (pk, ip.State)
)
validators = request.data.get('validators', {})
validate_xml_file = validators.get('validate_xml_file', False)
validate_file_format = validators.get('validate_file_format', False)
validate_integrity = validators.get('validate_integrity', False)
validate_logical_physical_representation = validators.get('validate_logical_physical_representation', False)
step = ProcessStep.objects.create(
name="Submit SIP",
information_package=ip
)
step.tasks.add(ProcessTask.objects.create(
name="preingest.tasks.UpdateIPStatus",
params={
"ip": ip,
"status": "Submitting",
},
processstep_pos=0,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
reception = Path.objects.get(entity="path_preingest_reception").value
sd_profile = ip.get_profile('submit_description')
container_format = ip.get_container_format()
container_file = os.path.join(reception, str(ip.pk) + ".%s" % container_format.lower())
sa = ip.SubmissionAgreement
info = sd_profile.fill_specification_data(sa, ip)
info["_IP_CREATEDATE"] = timestamp_to_datetime(creation_date(container_file)).isoformat()
infoxml = os.path.join(reception, str(ip.pk) + ".xml")
filesToCreate = {
infoxml: sd_profile.specification
}
step.tasks.add(ProcessTask.objects.create(
name="preingest.tasks.GenerateXML",
params={
"info": info,
"filesToCreate": filesToCreate,
"folderToParse": container_file,
"algorithm": ip.get_checksum_algorithm(),
},
processstep_pos=10,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
if validate_xml_file:
step.tasks.add(
ProcessTask.objects.create(
name="preingest.tasks.ValidateXMLFile",
params={
"xml_filename": infoxml
},
processstep_pos=14,
log=EventIP,
information_package=ip,
responsible=self.request.user,
)
)
if validate_file_format or validate_integrity:
step.tasks.add(
ProcessTask.objects.create(
name="preingest.tasks.ValidateFiles",
params={
"ip": ip,
"rootdir": reception,
"xmlfile": infoxml,
"validate_fileformat": validate_file_format,
"validate_integrity": validate_integrity,
},
processstep_pos=15,
log=EventIP,
information_package=ip,
responsible=self.request.user,
)
)
if validate_logical_physical_representation:
step.tasks.add(
ProcessTask.objects.create(
name="preingest.tasks.ValidateLogicalPhysicalRepresentation",
params={
"files": [os.path.basename(ip.ObjectPath)],
"xmlfile": infoxml,
},
processstep_pos=16,
log=EventIP,
information_package=ip,
responsible=self.request.user,
)
)
step.tasks.add(ProcessTask.objects.create(
name="preingest.tasks.SubmitSIP",
params={
"ip": ip
},
processstep_pos=20,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
if ip.get_email_recipient():
recipients = [ip.get_email_recipient()]
subject = request.data.get('subject')
body = request.data.get('body')
attachments = [ip.ObjectPath]
step.tasks.add(ProcessTask.objects.create(
name="ESSArch_Core.tasks.SendEmail",
params={
'sender': self.request.user.email,
'recipients': recipients,
'subject': subject,
'body': body,
'attachments': attachments
},
processstep_pos=25,
information_package=ip,
responsible=self.request.user
))
step.tasks.add(ProcessTask.objects.create(
name="preingest.tasks.UpdateIPStatus",
params={
"ip": ip,
"status": "Submitted"
},
processstep_pos=30,
log=EventIP,
information_package=ip,
responsible=self.request.user,
))
step.save()
step.run()
return Response({'status': 'submitting ip'})
def _run(self):
def get_information_packages(job):
return self.rule.information_packages.filter(
active=True, ).exclude(conversion_job_entries__job=self, )
ips = get_information_packages(self)
for ip in ips.order_by(
'-cached').iterator(): # convert cached IPs first
while not ip.cached:
with allow_join_result():
t, created = ProcessTask.objects.get_or_create(
name='workflow.tasks.CacheAIP',
information_package=ip,
defaults={
'responsible': ip.responsible,
'eager': False
})
if not created:
t.run()
time.sleep(10)
ip.refresh_from_db()
policy = ip.policy
srcdir = os.path.join(policy.cache_storage.value,
ip.object_identifier_value)
new_ip = ip.create_new_generation(ip.state, ip.responsible, None)
dstdir = os.path.join(policy.cache_storage.value,
new_ip.object_identifier_value)
new_ip.object_path = dstdir
new_ip.save()
aip_profile = new_ip.get_profile_rel('aip').profile
aip_profile_data = new_ip.get_profile_data('aip')
mets_dir, mets_name = find_destination("mets_file",
aip_profile.structure)
mets_path = os.path.join(srcdir, mets_dir, mets_name)
mets_tree = etree.parse(mets_path)
# copy files to new generation
shutil.copytree(srcdir, dstdir)
# convert files specified in rule
for pattern, spec in six.iteritems(self.rule.specification):
target = spec['target']
tool = spec['tool']
for path in iglob(dstdir + '/' + pattern):
if os.path.isdir(path):
for root, dirs, files in walk(path):
rel = os.path.relpath(root, dstdir)
for f in files:
fpath = os.path.join(root, f)
job_entry = ConversionJobEntry.objects.create(
job=self,
start_date=timezone.now(),
ip=ip,
old_document=os.path.join(rel, f))
convert_file(fpath, target)
os.remove(fpath)
job_entry.new_document = os.path.splitext(
job_entry.old_document)[0] + '.' + target
job_entry.end_date = timezone.now()
job_entry.tool = tool
job_entry.save()
elif os.path.isfile(path):
rel = os.path.relpath(path, dstdir)
job_entry = ConversionJobEntry.objects.create(
job=self,
start_date=timezone.now(),
ip=ip,
old_document=rel,
)
convert_file(path, target)
os.remove(path)
job_entry.new_document = os.path.splitext(
job_entry.old_document)[0] + '.' + target
job_entry.end_date = timezone.now()
job_entry.tool = tool
job_entry.save()
# preserve new generation
sa = new_ip.submission_agreement
try:
os.remove(mets_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
filesToCreate = OrderedDict()
try:
premis_profile = new_ip.get_profile_rel(
'preservation_metadata').profile
premis_profile_data = ip.get_profile_data(
'preservation_metadata')
except ProfileIP.DoesNotExist:
pass
else:
premis_dir, premis_name = find_destination(
"preservation_description_file", aip_profile.structure)
premis_path = os.path.join(dstdir, premis_dir, premis_name)
try:
os.remove(premis_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
filesToCreate[premis_path] = {
'spec':
premis_profile.specification,
'data':
fill_specification_data(premis_profile_data,
ip=new_ip,
sa=sa),
}
filesToCreate[mets_path] = {
'spec':
aip_profile.specification,
'data':
fill_specification_data(aip_profile_data, ip=new_ip, sa=sa),
}
t = ProcessTask.objects.create(
name='ESSArch_Core.tasks.GenerateXML',
params={
'filesToCreate': filesToCreate,
'folderToParse': dstdir,
},
responsible=new_ip.responsible,
information_package=new_ip,
)
t.run().get()
dsttar = dstdir + '.tar'
dstxml = dstdir + '.xml'
objid = new_ip.object_identifier_value
with tarfile.open(dsttar, 'w') as tar:
for root, dirs, files in walk(dstdir):
rel = os.path.relpath(root, dstdir)
for d in dirs:
src = os.path.join(root, d)
arc = os.path.join(objid, rel, d)
arc = os.path.normpath(arc)
index_path(new_ip, src)
tar.add(src, arc, recursive=False)
for f in files:
src = os.path.join(root, f)
index_path(new_ip, src)
tar.add(src,
os.path.normpath(os.path.join(objid, rel, f)))
algorithm = policy.get_checksum_algorithm_display()
checksum = calculate_checksum(dsttar, algorithm=algorithm)
info = fill_specification_data(
new_ip.get_profile_data('aip_description'), ip=new_ip, sa=sa)
info["_IP_CREATEDATE"] = timestamp_to_datetime(
creation_date(dsttar)).isoformat()
aip_desc_profile = new_ip.get_profile('aip_description')
filesToCreate = {
dstxml: {
'spec': aip_desc_profile.specification,
'data': info
}
}
ProcessTask.objects.create(
name="ESSArch_Core.tasks.GenerateXML",
params={
"filesToCreate": filesToCreate,
"folderToParse": dsttar,
"extra_paths_to_parse": [mets_path],
"algorithm": algorithm,
},
information_package=new_ip,
responsible=new_ip.responsible,
).run().get()
InformationPackage.objects.filter(pk=new_ip.pk).update(
message_digest=checksum,
message_digest_algorithm=policy.checksum_algorithm,
)
ProcessTask.objects.create(
name='ESSArch_Core.tasks.UpdateIPSizeAndCount',
information_package=new_ip,
responsible=new_ip.responsible,
).run().get()
t = ProcessTask.objects.create(
name='workflow.tasks.StoreAIP',
information_package=new_ip,
responsible=new_ip.responsible,
)
t.run()
def list(self, request):
reception = Path.objects.get(entity="path_ingest_reception").value
uip = Path.objects.get(entity="path_ingest_unidentified").value
ips = []
for xmlfile in glob.glob(os.path.join(reception, "*.xml")) + glob.glob(os.path.join(uip, "*.xml")):
if os.path.isfile(xmlfile):
if xmlfile.startswith(uip):
srcdir = uip
else:
srcdir = reception
ip = self.parseFile(xmlfile, srcdir)
if not InformationPackage.objects.filter(id=ip['id']).exists():
ips.append(ip)
for container_file in glob.glob(os.path.join(uip, "*.tar")) + glob.glob(os.path.join(uip, "*.zip")):
ip = {
'Label': os.path.basename(container_file),
'CreateDate': str(timestamp_to_datetime(creation_date(container_file)).isoformat()),
'State': 'Unidentified',
'status': 0,
'step_state': celery_states.SUCCESS,
}
include = True
for xmlfile in glob.glob(os.path.join(uip, "*.xml")):
if os.path.isfile(xmlfile):
doc = etree.parse(xmlfile)
root = doc.getroot()
el = root.xpath('.//*[local-name()="%s"]' % "FLocat")[0]
if ip['Label'] == get_value_from_path(el, "@href").split('file:///')[1]:
include = False
break
if include:
ips.append(ip)
from_db = InformationPackage.objects.filter(State='Receiving').prefetch_related(
Prefetch('profileip_set', to_attr='profiles'),
)
serializer = InformationPackageSerializer(
data=from_db, many=True, context={'request': request}
)
serializer.is_valid()
ips.extend(serializer.data)
try:
ordering = request.query_params.get('ordering', '')
reverse = ordering.startswith('-')
ordering = remove_prefix(ordering, '-')
ips = sorted(ips, key=lambda k: k[ordering], reverse=reverse)
except KeyError:
pass
paginator = LinkHeaderPagination()
page = paginator.paginate_queryset(ips, request)
if page is not None:
return paginator.get_paginated_response(page)
return Response(ips)