def get_651(key):
found = []
for src in get_src(key):
data = get_from_local(src)
for tag, line in get_tag_lines(data, ['651']):
found.append(list(get_all_subfields(line)))
return found
def process_record(pos, loc, data):
for tag in '100', '700':
line = get_first_tag(data, set([tag]))
if line:
fields = list(get_all_subfields(line))
if any(k == 'c' for k, v in fields):
print((loc, fields))
def process_record(pos, loc, data):
for tag in '100', '700':
line = get_first_tag(data, set([tag]))
if line:
fields = list(get_all_subfields(line))
if any(k == 'c' for k, v in fields):
print((loc, fields))
def get_651(key):
found = []
for src in get_src(key):
data = get_from_local(src)
for tag, line in get_tag_lines(data, ['651']):
found.append(list(get_all_subfields(line)))
return found
from catalog.utils.query import query_iter, set_staging, withKey, get_mc
import sys, codecs, re
sys.path.append('/home/edward/src/olapi')
from olapi import OpenLibrary, Reference
from catalog.read_rc import read_rc
from catalog.get_ia import get_from_archive, get_from_local
from catalog.marc.fast_parse import get_first_tag, get_all_subfields
rc = read_rc()
sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
set_staging(True)
ol = OpenLibrary("http://dev.openlibrary.org")
ol.login('EdwardBot', rc['EdwardBot'])
q = { 'type': '/type/edition', 'table_of_contents': None, 'subjects': None }
queue = []
count = 0
for e in query_iter(q, limit=100):
key = e['key']
mc = get_mc(key)
if not mc:
continue
data = get_from_local(mc)
line = get_first_tag(data, set(['041']))
if not line:
continue
print key, line[0:2], list(get_all_subfields(line))
def marc_authors(data):
line = get_first_tag(data, set(['100', '110', '111']))
return ''.join("<b>$%s</b>%s" % (esc(k), esc(v))
for k, v in get_all_subfields(line)) if line else None
from catalog.utils.query import query_iter, set_staging, withKey, get_mc
import sys, codecs, re
sys.path.append('/home/edward/src/olapi')
from olapi import OpenLibrary, Reference
from catalog.read_rc import read_rc
from catalog.get_ia import get_from_archive, get_from_local
from catalog.marc.fast_parse import get_first_tag, get_all_subfields
rc = read_rc()
sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
set_staging(True)
ol = OpenLibrary("http://dev.openlibrary.org")
ol.login('EdwardBot', rc['EdwardBot'])
q = {'type': '/type/edition', 'table_of_contents': None, 'subjects': None}
queue = []
count = 0
for e in query_iter(q, limit=100):
key = e['key']
mc = get_mc(key)
if not mc:
continue
data = get_from_local(mc)
line = get_first_tag(data, set(['041']))
if not line:
continue
print key, line[0:2], list(get_all_subfields(line))
def marc_publisher(data):
line = get_first_tag(data, set(['260']))
return ''.join("<b>$%s</b>%s" % (esc(k), esc(v))
for k, v in get_all_subfields(line)) if line else None
locs = set()
for ek in edition_keys:
e = site.withKey(ek)
for i in e.isbn_10 if e.isbn_10 else []:
locs.update(search_query('isbn', i))
for i in e.lccn if e.lccn else []:
locs.update(search_query('lccn', i))
for i in e.oclc_numbers if e.oclc_numbers else []:
locs.update(search_query('oclc', i))
print len(locs), 'MARC records found'
def ldv(line):
for s in ('1452', '1519', 'eonard', 'inci'):
if line.find(s) != -1:
return True
return False
for loc in locs:
# print loc
data = get_data(loc)
if not data:
print "couldn't get"
continue
line = get_first_tag(data, set(['100', '110', '111']))
if line and ldv(line):
print list(get_all_subfields(line))
line = get_first_tag(data, set(['700', '710', '711']))
if line and ldv(line):
print list(get_all_subfields(line))
locs = set()
for ek in edition_keys:
e = site.withKey(ek)
for i in e.isbn_10 if e.isbn_10 else []:
locs.update(search_query('isbn', i))
for i in e.lccn if e.lccn else []:
locs.update(search_query('lccn', i))
for i in e.oclc_numbers if e.oclc_numbers else []:
locs.update(search_query('oclc', i))
print len(locs), 'MARC records found'
def ldv(line):
for s in ('1452', '1519', 'eonard', 'inci'):
if line.find(s) != -1:
return True
return False
for loc in locs:
# print loc
data = get_data(loc)
if not data:
print "couldn't get"
continue
line = get_first_tag(data, set(['100', '110', '111']))
if line and ldv(line):
print list(get_all_subfields(line))
line = get_first_tag(data, set(['700', '710', '711']))
if line and ldv(line):
print list(get_all_subfields(line))
def marc_authors(data):
line = get_first_tag(data, set(["100", "110", "111"]))
return "".join("<b>$%s</b>%s" % (esc(k), esc(v)) for k, v in get_all_subfields(line)) if line else None
from catalog.marc.fast_parse import get_all_subfields
import re
samples = [
"00\x1faDi 1 juan.Guo se tian xiang /Wu Jingsuo bian.Ba duan jin /Xingshizhushi bian ji --di 2 juan.Wu sheng xi ;Shi er lou /Li Yu --di 3 juan.Jin xiang ting /Su'anzhuren bian.\x1ftFen zhuang lou /Zhuxishanren --\x1fgdi 4 juan.Wu se shi /Bilian'gezhuren.Ba dong tian /Wuseshizhuren.Wu feng yin /Chichi dao ren bian zhu --di 5 juan.Shuang feng qi yuan /Xueqiaozhuren zi ding.Jin shi yuan.Qing meng tuo /Anyangjiumin --di 6 juan.Wu mei yuan.Xiu qiu yuan.Yuan yang ying /Qiaoyunshanren bian --di 7 juan.Mei ren shu /Xu Zhen.Wan hua lou /Li Yutang --di 8 juan.Bei shi yan yi /Du Gang.Kong kong huan /Wugangzhuren bian ci.Chun qiu pei --di 9 juan.Qian Qi guo zhi /Wumenxiaoke.Hou Qi guo zhi /Yanshuisanren.Qiao shi yan yi /Lu Yingyang --di 10 juan.Liaohai dan zhong lu /Lu Renlong.Tian bao tu.Jin xiu yi --di 11 juan.Shi mei tu.Huan xi yuan jia /Xihuyuyinzhuren.Feng liu he shang.Liang jiao hun /Tianhuazangzhuren --di 12 juan.Ge lian hua ying.Qi lou chong meng /Wang Lanzhi.\x1e",
'00\x1ftManierismus als Artistik : systematische Aspekte einer \xe8asthetischen Kategorie / R\xe8udiger Zymner -- "Stil" und "Manier" in der Alltagskultur / Kaspar Maase -- Die Wortfamilie von it. "Maniera" zwischen Literatur, bildender Kunst und Psychologie / Margarete Lindemann -- Der Manierismus : zur Problematik einer kunsthistorischen Erfindung / Horst Bredekamp -- Inszenierte K\xe8unstlichkeit : Musik als manieristisches Dispositiv / Hermann Danuser -- Manierismus als Stilbegriff in der Architekturgeschichte / Hermann Hipp -- "Raffael ohne H\xe8ande," oder, Das Kunstwerk zwischen Sch\xe8opfung und Fabrikation : Konzepte der "maniera" bei Vasari und seinen Zeitgenossen / Ursula Link-Heer -- "Sprezzatura" : Pontormos Portraits und das h\xe8ofische Ideal des Manierismus / Axel Christoph Gampp -- Maniera and the grotesque / Maria Fabricius Hansen -- Neulateinisches Figurengedicht und manieristische Poetik : zum "Poematum liber" (1573) des Richard Willis / Ulrich Ernst -- Manierismus als Selbstbehauptung, Jean Paul / Wolfgang Braungart -- Artistische Erkenntnis : (Sprach-)Alchimie und Manierismus in der Romantik / Axel Dunker -- "Als lebeten sie" / Holk Cruse.\x1e',
]
re_gt = re.compile('^(gt)+$')
re_gtr = re.compile('^(gtr)+$')
re_at = re.compile('^at+$')
re_end_num = re.compile('\d[]. ]*$')
for line in open('test_data/marc_toc'):
(loc, line) = eval(line)
#print loc
subfields = list(get_all_subfields(line))
if subfields[0][0] == '6':
subfields.pop(0)
subtags = ''.join(k for k, v in subfields)
if re_at.match(subtags):
a = subfields[0][1]
m = re_end_num.search(a)
print bool(m), ` a `
continue
if not m:
for k, v in subfields:
print k, ` v `
assert m
continue
if re_gtr.match(subtags):
def marc_publisher(data):
line = get_first_tag(data, set(['260']))
return ''.join("<b>$%s</b>%s" % (esc(k), esc(v)) for k, v in get_all_subfields(line)) if line else None
for name, part, size in files():
f = open(name)
print part
for pos, loc, data in read_marc_file(part, f):
if str(data)[6:8] != 'am': # only want books
continue
books += 1
line = get_first_tag(data, set(['041']))
if not line:
continue
has_041 += 1
if line[0] == '0':
i1_0 += 1
if line[0] == '1':
i1_1 += 1
subfields = list(get_all_subfields(line))
print loc
keys = get_keys(loc)
print keys, line[0:2], subfields
continue
if line[1] != ' ':
i2 += 1
print 'i2:', line[0:2], subfields
if '\x1fa' in line:
has_a +=1
else:
print 'no a:', line[0:2], subfields
if '\x1fh' in line:
has_h +=1
if '\x1f2' in line:
has_2 +=1
