def main(file_path, file_uuid, sip_uuid):
failed = False
# Get file format
try:
fmt = FormatVersion.active.get(fileformatversion__file_uuid=file_uuid)
except FormatVersion.DoesNotExist:
rules = fmt = None
if fmt:
rules = FPRule.active.filter(format=fmt.uuid, purpose='validation')
# Check for a default rule exists
if not rules:
rules = FPRule.active.filter(purpose='default_validation')
for rule in rules:
if rule.command.script_type in ('bashScript', 'command'):
command_to_execute = replace_string_values(rule.command.command,
file_=file_uuid, sip=sip_uuid, type_='file')
args = []
else:
command_to_execute = rule.command.command
args = [file_path]
print('Running', rule.command.description)
exitstatus, stdout, stderr = executeOrRun(rule.command.script_type,
command_to_execute, arguments=args)
if exitstatus != 0:
print('Command {} failed with exit status {}; stderr:'.format(rule.command.description, exitstatus),
stderr, file=sys.stderr)
failed = True
continue
print('Command {} completed with output {}'.format(rule.command.description, stdout))
# Parse output and generate an Event
# Output is JSON in format:
# { "eventOutcomeInformation": "pass",
# "eventOutcomeDetailNote": "format=\"JPEG\"; version=\"1.01\"; result=\"Well-Formed and valid\"" }
# Or
# { "eventOutcomeInformation": "fail",
# "eventOutcomeDetailNote": "format=\"Not detected\"; result=\"Not well-formed\"" }
output = ast.literal_eval(stdout)
event_detail = 'program="{tool.description}"; version="{tool.version}"'.format(tool=rule.command.tool)
print('Creating validation event for {} ({})'.format(file_path, file_uuid))
databaseFunctions.insertIntoEvents(
fileUUID=file_uuid,
eventType='validation',
eventDetail=event_detail,
eventOutcome=output.get('eventOutcomeInformation'),
eventOutcomeDetailNote=output.get('eventOutcomeDetailNote'),
)
if failed:
return -1
else:
return 0
def _get_command_to_execute(self, rule):
"""Return a 2-tuple consisting of a) the FPR rule ``rule``'s command
and b) a list of arguments to pass to it.
"""
if rule.command.script_type in ('bashScript', 'command'):
return (replace_string_values(rule.command.command,
file_=self.file_uuid,
sip=self.sip_uuid, type_='file'),
[])
else:
return (rule.command.command, [self.file_path, self.policies_dir])
def main(job, file_path, file_uuid, sip_uuid):
setup_dicts(mcpclient_settings)
failed = False
# Check to see whether the file has already been characterized; don't try
# to characterize it a second time if so.
if FPCommandOutput.objects.filter(file_id=file_uuid).count() > 0:
return 0
try:
format = FormatVersion.active.get(
fileformatversion__file_uuid=file_uuid)
except FormatVersion.DoesNotExist:
rules = format = None
if format:
rules = FPRule.active.filter(format=format.uuid,
purpose="characterization")
# Characterization always occurs - if nothing is specified, get one or more
# defaults specified in the FPR.
if not rules:
rules = FPRule.active.filter(purpose="default_characterization")
for rule in rules:
if (rule.command.script_type == "bashScript"
or rule.command.script_type == "command"):
args = []
command_to_execute = replace_string_values(rule.command.command,
file_=file_uuid,
sip=sip_uuid,
type_="file")
else:
rd = ReplacementDict.frommodel(file_=file_uuid,
sip=sip_uuid,
type_="file")
args = rd.to_gnu_options()
command_to_execute = rule.command.command
exitstatus, stdout, stderr = executeOrRun(
rule.command.script_type,
command_to_execute,
arguments=args,
capture_output=True,
)
job.write_output(stdout)
job.write_error(stderr)
if exitstatus != 0:
job.write_error(
"Command {} failed with exit status {}; stderr:".format(
rule.command.description, exitstatus))
failed = True
continue
# fmt/101 is XML - we want to collect and package any XML output, while
# allowing other commands to execute without actually collecting their
# output in the event that they are writing their output to disk.
# FPCommandOutput can have multiple rows for a given file,
# distinguished by the rule that produced it.
if (rule.command.output_format
and rule.command.output_format.pronom_id == "fmt/101"):
try:
etree.fromstring(stdout)
insertIntoFPCommandOutput(file_uuid, stdout, rule.uuid)
job.write_output(
'Saved XML output for command "{}" ({})'.format(
rule.command.description, rule.command.uuid))
except etree.XMLSyntaxError:
failed = True
job.write_error(
'XML output for command "{}" ({}) was not valid XML; not saving to database'
.format(rule.command.description, rule.command.uuid))
else:
job.write_error(
'Tool output for command "{}" ({}) is not XML; not saving to database'
.format(rule.command.description, rule.command.uuid))
if failed:
return 255
else:
return 0
def _execute_rule_command(self, rule):
"""Run the command against the file and return either 'passed' or
'failed'. If the command errors or determines that the file is invalid,
return 'failed'. Non-errors will result in the creation of an Event
model in the db. Preservation derivative validation will result in the
stdout from the command being saved to disk within the unit (i.e., SIP).
"""
result = 'passed'
if rule.command.script_type in ('bashScript', 'command'):
command_to_execute = replace_string_values(
rule.command.command,
file_=self.file_uuid,
sip=self.sip_uuid,
type_='file')
args = []
else:
command_to_execute = rule.command.command
args = [self.file_path]
self.job.print_output('Running', rule.command.description)
exitstatus, stdout, stderr = executeOrRun(
type=rule.command.script_type,
text=command_to_execute,
printing=False,
arguments=args)
if exitstatus != 0:
self.job.print_error(
'Command {description} failed with exit status {status};'
' stderr:'.format(description=rule.command.description,
status=exitstatus))
return 'failed'
# Parse output and generate an Event
# TODO: Evaluating a python string from a user-definable script seems
# insecure practice; should be JSON.
output = ast.literal_eval(stdout)
event_detail = ('program="{tool.description}";'
' version="{tool.version}"'.format(
tool=rule.command.tool))
# If the FPR command has not errored but the actual validation
# determined that the file is not valid, then we want to both create a
# validation event in the db and set ``failed`` to ``True`` because we
# want the micro-service in the dashboard GUI to indicate "Failed".
# NOTE: this requires that the stdout of all validation FPR commands be
# a dict (preferably a JSON object) with an ``eventOutcomeInformation``
# boolean attribute.
if output.get('eventOutcomeInformation') == 'pass':
self.job.print_output('Command "{}" was successful'.format(rule.command.description))
else:
self.job.pyprint('Command {cmd_description} indicated failure with this'
' output:\n\n{output}'.format(
cmd_description=rule.command.description,
output=pformat(stdout)),
file=sys.stderr)
result = 'failed'
if self.file_type == 'preservation':
self._save_stdout_to_logs_dir(output)
self.job.print_output('Creating {purpose} event for {file_path} ({file_uuid})'.format(
purpose=self.purpose,
file_path=self.file_path,
file_uuid=self.file_uuid))
databaseFunctions.insertIntoEvents(
fileUUID=self.file_uuid,
eventType='validation', # From PREMIS controlled vocab.
eventDetail=event_detail,
eventOutcome=output.get('eventOutcomeInformation'),
eventOutcomeDetailNote=output.get('eventOutcomeDetailNote'),
)
return result
