def process(self):
# foo = processing.ListProgress2(self, self.data['package'])
# foo.logger = lambda x: self.logger.log("Zipping : {}".format(x))
# lambda l: self.logger.log("{}{}".format(datetime.datetime.now(), l)))
if self.data['workflow'] == "DS":
processing_workflow = workflow.Workflow(workflow.DSWorkflow())
elif self.data['workflow'] == "BrittleBooks":
processing_workflow = workflow.Workflow(
workflow.BrittleBooksWorkflow())
else:
raise Exception("Unknown workflow, {}".format(
self.data['workflow']))
tasks = processing_workflow.zip(self.data['package'],
self.data['export_destination'])
processing_window = processing.ListCallableProgress(
self, tasks=tasks, task_name="Zipping")
processing_window.logger = self.logger.log
try:
self.logger.log("Zipping process started")
processing_window.process()
self.logger.log("Zipping completed.")
self.logger.log("Files can be found at {}".format(
self.data['export_destination']))
except processing.ProcessCanceled:
return False
return True
def process(self):
if self.data['workflow'] == "DS":
processing_workflow = workflow.Workflow(workflow.DSWorkflow())
elif self.data['workflow'] == "BrittleBooks":
processing_workflow = workflow.Workflow(
workflow.BrittleBooksWorkflow())
else:
raise Exception("Unknown workflow, {}".format(
self.data['workflow']))
tasks = processing_workflow.validate(self.data['package'])
processing_window = processing.ListCallableProgress(
self, tasks=tasks, task_name="Validating")
processing_window.logger = self.logger.log
try:
self.logger.log("Validation started")
processing_window.process()
self.logger.log("Validation completed")
except processing.ProcessCanceled:
return False
message = self.build_report(processing_window.results)
self.logger.log(message)
return True
def test_ds_item_len(ds_collection_root):
strategy = workflow.DSWorkflow()
package_builder = workflow.Workflow(strategy)
my_collection = package_builder.build_package(ds_collection_root)
my_object = my_collection[0]
my_item = my_object[0]
assert len(my_item) == 1
def test_ds_instantiations(ds_collection_root):
strategy = workflow.DSWorkflow()
package_builder = workflow.Workflow(strategy)
my_package = package_builder.build_package(ds_collection_root)
my_object = my_package[0]
my_item = my_object[0]
my_instantiation = my_item.instantiations["access"]
assert my_instantiation.metadata['category'] == "access"
pprint(dict(my_instantiation.metadata))
print(my_instantiation)
def __init__(self, parent=None):
super().__init__(parent)
self.data = {
"workflow": "DS"
}
self.addPage(HathiWizardPages["PackageBrowser"].wizard_page(self))
self.addPage(HathiWizardPages["EndPage"].wizard_page(self))
self.data["root"] = ROOT
workflow_strat = workflow.DSWorkflow()
# package_builder = collection_builder.BuildPackage(workflow_strat)
package_builder = workflow.Workflow(workflow_strat)
self.data["package"] = package_builder.build_package(ROOT)
def build_package(self, root):
if self.data['workflow'] == "DS":
workflow_strats = workflow.DSWorkflow()
elif self.data['workflow'] == "BrittleBooks":
workflow_strats = workflow.BrittleBooksWorkflow()
else:
raise Exception("Unknown workflow {}".format(
self.data['workflow']))
# package_builder = collection_builder.BuildPackage(workflow_strats)
package_builder = workflow.Workflow(workflow_strats)
print("Loading")
package_locator = LocatingPackagesDialog(package_builder, root)
package_locator.exec_()
# self.data
return package_locator.package
def process(self):
self.logger.log("Processing")
if self.data['workflow'] == "DS":
processing_workflow = workflow.Workflow(workflow.DSWorkflow())
else:
raise Exception("invalid workflow, {}".format(
self.data['workflow']))
tasks = processing_workflow.prep(self.data['package'])
processing_window = processing.ListCallableProgress(
self, tasks=tasks, task_name="Prepping")
processing_window.logger = self.logger.log
try:
self.logger.log("Prep started")
processing_window.process()
self.logger.log("Prep completed")
except processing.ProcessCanceled:
return False
return True
def process(self):
self.logger.log("Processing")
if self.data['workflow'] == "BrittleBooks":
processing_workflow = workflow.Workflow(
workflow.BrittleBooksWorkflow())
else:
raise Exception("invalid workflow, {}".format(
self.data['workflow']))
tasks = processing_workflow.update_checksums(self.data['package'])
processing_window = processing.ListCallableProgress(
self, tasks=tasks, task_name="Updating checksums")
processing_window.logger = self.logger.log
try:
self.logger.log("Updating checksum started")
processing_window.process()
self.logger.log("Updating checksum completed")
except processing.ProcessCanceled:
return False
return True
def process(self):
self.logger.log("Processing")
if self.data['workflow'] == "DS":
processing_workflow = workflow.Workflow(workflow.DSWorkflow())
else:
raise Exception("invalid workflow, {}".format(
self.data['workflow']))
tasks = processing_workflow.qc(self.data['package'])
processing_window = processing.ListCallableProgress(
self, tasks=tasks, task_name="Performing QC")
try:
self.logger.log("QC started")
processing_window.process()
report = self._create_report(processing_window.results)
self.logger.log("QC ended")
self.logger.log("{}\n".format(report))
except processing.ProcessCanceled:
return False
return True
def test_prep(ds_collection):
ds_workflow = workflow.Workflow(workflow.DSWorkflow())
new_package = ds_workflow.build_package(ds_collection)
new_package.children[0].component_metadata["title_page"] = "00000002.jp2"
new_package.children[1].component_metadata["title_page"] = "00000002.jp2"
new_package.children[2].component_metadata["title_page"] = "00000002.jp2"
for task in ds_workflow.prep(new_package):
task()
prepped_package = ds_workflow.build_package(new_package.path)
errors = []
for package_object in prepped_package:
path = package_object.metadata['path']
meta = os.path.join(path, "meta.yml")
assert os.path.exists(meta)
with open(meta, "r") as f:
yaml = f.read()
if """ 00000002.jp2:
label: TITLE""" not in yaml:
pytest.fail("missing title_page in {}".format(meta))
tasks = ds_workflow.validate(prepped_package)
for task in tasks:
errors += task()
for error in errors:
print(error)
assert len(errors) == 0
with tempfile.TemporaryDirectory() as temp_destination:
for task in ds_workflow.zip(prepped_package, temp_destination):
task()
assert os.path.exists(os.path.join(temp_destination, "1564651.zip"))
assert os.path.exists(os.path.join(temp_destination, "4564654.zip"))
assert os.path.exists(os.path.join(temp_destination, "7213538.zip"))
def test_ds_object_id(ds_collection_root):
strategy = workflow.DSWorkflow()
package_builder = workflow.Workflow(strategy)
my_collection = package_builder.build_package(ds_collection_root)
my_object = my_collection[0]
assert my_object.metadata['id'] == "1564651"
def test_ds_package_len(ds_collection_root):
strategy = workflow.DSWorkflow()
package_builder = workflow.Workflow(strategy)
my_collection = package_builder.build_package(ds_collection_root)
assert len(my_collection) == 3
def test_ds_collection_strategy(ds_collection_root):
strategy = workflow.DSWorkflow()
package_builder = workflow.Workflow(strategy)
my_collection = package_builder.build_package(ds_collection_root)
assert isinstance(my_collection, collection.Package)
def test_bb_collection_strat(bb_collection_root):
strategy = workflow.BrittleBooksWorkflow()
# strategy = collection_builder.BrittleBooksStrategy()
package_builder = workflow.Workflow(strategy)
my_collection = package_builder.build_package(bb_collection_root)
assert isinstance(my_collection, collection.Package)
def test_full_workflow(bb_collection):
print(bb_collection)
my_workflow = workflow.Workflow(workflow.BrittleBooksWorkflow())
new_package = my_workflow.build_package(bb_collection)
for package_object in new_package:
with open(os.path.join(package_object.metadata['path'], "marc.xml"),
"w") as f:
f.write(
'\n<record xmlns="http://www.loc.gov/MARC21/slim" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">'
)
f.write('\n <leader>00616nam a2200181 a 4500</leader>')
f.write('\n <controlfield tag="001">1251150</controlfield>')
f.write(
'\n <controlfield tag="005">20020415162036.0</controlfield>')
f.write(
'\n <controlfield tag="008">840418s1913 fr 00010 fre d</controlfield>'
)
f.write('\n <datafield tag="035" ind1=" " ind2=" ">')
f.write('\n <subfield code="a">(OCoLC)ocm10643170</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="035" ind1=" " ind2=" ">')
f.write('\n <subfield code="9">AHI-6903</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="040" ind1=" " ind2=" ">')
f.write('\n <subfield code="a">ORC</subfield>')
f.write('\n <subfield code="c">ORC</subfield>')
f.write('\n <subfield code="d">UIU</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="043" ind1=" " ind2=" ">')
f.write('\n <subfield code="a">e-fr---</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="100" ind1="1" ind2="0">')
f.write('\n <subfield code="a">Doumic, Rene,</subfield>')
f.write('\n <subfield code="d">1860-1937</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="245" ind1="1" ind2="0">')
f.write('\n <subfield code="a">De Scribe Ibsen :</subfield>')
f.write(
'\n <subfield code="b">causeries sur le tcontemporain : ouvrage r Francaise /</subfield>'
)
f.write('\n <subfield code="c">Ree Doumic.</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="260" ind1="0" ind2=" ">')
f.write('\n <subfield code="a">Paris :</subfield>')
f.write('\n <subfield code="b">Perrin,</subfield>')
f.write('\n <subfield code="c">1913.</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="300" ind1=" " ind2=" ">')
f.write('\n <subfield code="a">xvi, 352 p. ;</subfield>')
f.write('\n <subfield code="c">19 cm.</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="650" ind1=" " ind2="0">')
f.write('\n <subfield code="a">Drama</subfield>')
f.write(
'\n <subfield code="x">History and criticism</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="650" ind1=" " ind2="0">')
f.write('\n <subfield code="a">French drama</subfield>')
f.write('\n <subfield code="y">19th century</subfield>')
f.write(
'\n <subfield code="x">History and criticism.</subfield>')
f.write('\n </datafield>')
f.write('\n <datafield tag="955" ind1=" " ind2=" ">')
f.write('\n <subfield code="b">1251150</subfield>')
f.write('\n </datafield>')
f.write('\n</record>')
f.write('\n<?query @attr 1=12 "1251150" ?>')
f.write('\n<?count 1 ?>')
for task in my_workflow.update_checksums(new_package):
task()
errors = []
for task in my_workflow.validate(new_package):
errors += task()
for error in errors:
print(error)
assert len(errors) == 0