/
philologic_missing_images.py
74 lines (46 loc) · 2.04 KB
/
philologic_missing_images.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
## Generates a report for missing Philologic images
import sys, urllib2, csv
from eulfedora.server import Repository
from bs4 import BeautifulSoup
from collections import defaultdict
HOST = 'http://localhost:8080'
fedoraUser = 'xxxxx'
fedoraPass = 'xxxxx'
def main(argv):
repo = Repository(root='%s/fedora/' % HOST, username='%s' % fedoraUser, password='%s' % fedoraPass)
philologic_pids = repo.get_objects_with_cmodel(cmodel_uri='info:fedora/niu-objects:cmodel')
phil_doc = open('phil_doc.csv', 'w')
image_ids = []
d = defaultdict(int)
for p in philologic_pids:
philologic = p.getDatastreamObject('OBJ').content
substring = 'ARTFL-figure-missing'
if substring in philologic:
print 'Processing %s' % p
images = []
image_count = 0
url = '%s/fedora/objects/%s/datastreams/OBJ/content' % (HOST, p)
passwordManager = urllib2.HTTPPasswordMgrWithDefaultRealm()
fedoraAdmin = "%s/fedora" % HOST
passwordManager.add_password(None, fedoraAdmin, fedoraUser, fedoraPass)
handler = urllib2.HTTPBasicAuthHandler(passwordManager)
fedoraOpener = urllib2.build_opener(handler)
soup = BeautifulSoup(fedoraOpener.open(url), 'html.parser')
spans = soup.find_all('span', 'ARTFL-figure-missing')
for span in spans:
image = span['sysid']
images.append(image)
image_count+= 1
image_ids.extend(images)
images_string = ';'.join(images)
phil_doc.write('%s,%s,%s\n' % (p, image_count, images_string))
print 'Successfully processed %s' % p
for i in image_ids:
d[i] += 1
with open('phil_image.csv', 'w') as outfile:
phil_image = csv.writer(outfile)
for key, value in d.items():
phil_image.writerow([key, value])
phil_doc.close()
if __name__ == '__main__':
sys.exit(main(sys.argv))