def update_keys(keys):
if not keys:
return 0
# FIXME: Some kind of hack introduced to work around DB connectivity issue
global args
logger.debug("Args: %s" % str(args))
update_work.load_configs(args.ol_url, args.config, 'default')
keys = [
k for k in keys
if k.count("/") == 2 and k.split("/")[1] in ("books", "authors",
"works")
]
count = 0
for chunk in web.group(keys, 100):
chunk = list(chunk)
count += len(chunk)
update_work.do_updates(chunk)
if count:
logger.info("updated %d documents", count)
return count
def update_keys(keys):
if not keys:
return 0
# FIXME: Some kind of hack introduced to work around DB connectivity issue
global args
logger.debug("Args: %s" % str(args))
update_work.load_configs(args['ol_url'], args['ol_config'], 'default')
keys = [
k for k in keys
if k.count("/") == 2 and k.split("/")[1] in ("books", "authors",
"works")
]
count = 0
for chunk in web.group(keys, 100):
chunk = list(chunk)
count += len(chunk)
update_work.do_updates(chunk)
# Caches should not persist between different calls to update_keys!
update_work.data_provider.clear_cache()
if count:
logger.info("updated %d documents", count)
return count
def submit_update_to_solr(target):
global sub_count
seq = int(math.ceil(len(target)/float(CHUNK_SIZE)))
chunks = [ target[i::seq] for i in xrange(seq) ]
for chunk in chunks:
update_work.load_configs(options.server,options.config,'default')
logger.info("Request %s/%s to update works: %s",str(sub_count),str(CHUNKS_NUM),str(chunk))
time.sleep(1)
update_work.do_updates(chunk)
sub_count = sub_count + 1
if (sub_count >= CHUNKS_NUM):
commit_it()
sub_count = 0
return 1
def submit_update_to_solr(target):
'''Executes the update queries for every element in the taget list.'''
global sub_count
seq = int(math.ceil(len(target)/float(CHUNK_SIZE)))
chunks = [ target[i::seq] for i in xrange(seq) ]
for chunk in chunks:
update_work.load_configs(options.server,options.config,'default')
logger.info("Request %s/%s to update works: %s",str(sub_count),str(CHUNKS_NUM),str(chunk))
time.sleep(1)
update_work.do_updates(chunk)
sub_count = sub_count + 1
if (sub_count >= CHUNKS_NUM):
commit_it()
sub_count = 0
return 1
def update_keys(keys):
global args
keys = (k for k in keys if k.count("/") == 2 and k.split("/")[1] in ["books", "authors", "works"])
update_work.clear_monkeypatch_cache(max_size=10000)
print str(args)
update_work.load_configs(args.ol_url, args.config, 'default')
count = 0
for chunk in web.group(keys, 100):
chunk = list(chunk)
count += len(chunk)
update_work.do_updates(chunk)
if count:
logger.info("updated %d documents", count)
return count
def update_keys(keys):
global args
keys = (k for k in keys if k.count("/") == 2 and k.split("/")[1] in ["books", "authors", "works"])
update_work.clear_monkeypatch_cache(max_size=10000)
print str(args)
update_work.load_configs(args.ol_url,args.config,'default')
count = 0
for chunk in web.group(keys, 100):
chunk = list(chunk)
count += len(chunk)
update_work.do_updates(chunk)
# update_work.update_keys(chunk, commit=False)
if count:
logger.info("updated %d documents", count)
return count
def update_keys(keys):
if not keys:
return 0
# FIXME: Some kind of hack introduced to work around DB connectivity issue
global args
logger.debug("Args: %s" % str(args))
update_work.load_configs(args.ol_url, args.config, 'default')
keys = (k for k in keys if k.count("/") == 2 and k.split("/")[1] in ["books", "authors", "works"])
update_work.clear_monkeypatch_cache(max_size=10000)
count = 0
for chunk in web.group(keys, 100):
chunk = list(chunk)
count += len(chunk)
update_work.do_updates(chunk)
if count:
logger.info("updated %d documents", count)
return count