def update_keys(keys): if not keys: return 0 # FIXME: Some kind of hack introduced to work around DB connectivity issue global args logger.debug("Args: %s" % str(args)) update_work.load_configs(args.ol_url, args.config, 'default') keys = [ k for k in keys if k.count("/") == 2 and k.split("/")[1] in ("books", "authors", "works") ] count = 0 for chunk in web.group(keys, 100): chunk = list(chunk) count += len(chunk) update_work.do_updates(chunk) if count: logger.info("updated %d documents", count) return count
def update_keys(keys): if not keys: return 0 # FIXME: Some kind of hack introduced to work around DB connectivity issue global args logger.debug("Args: %s" % str(args)) update_work.load_configs(args['ol_url'], args['ol_config'], 'default') keys = [ k for k in keys if k.count("/") == 2 and k.split("/")[1] in ("books", "authors", "works") ] count = 0 for chunk in web.group(keys, 100): chunk = list(chunk) count += len(chunk) update_work.do_updates(chunk) # Caches should not persist between different calls to update_keys! update_work.data_provider.clear_cache() if count: logger.info("updated %d documents", count) return count
def submit_update_to_solr(target): global sub_count seq = int(math.ceil(len(target)/float(CHUNK_SIZE))) chunks = [ target[i::seq] for i in xrange(seq) ] for chunk in chunks: update_work.load_configs(options.server,options.config,'default') logger.info("Request %s/%s to update works: %s",str(sub_count),str(CHUNKS_NUM),str(chunk)) time.sleep(1) update_work.do_updates(chunk) sub_count = sub_count + 1 if (sub_count >= CHUNKS_NUM): commit_it() sub_count = 0 return 1
def submit_update_to_solr(target): '''Executes the update queries for every element in the taget list.''' global sub_count seq = int(math.ceil(len(target)/float(CHUNK_SIZE))) chunks = [ target[i::seq] for i in xrange(seq) ] for chunk in chunks: update_work.load_configs(options.server,options.config,'default') logger.info("Request %s/%s to update works: %s",str(sub_count),str(CHUNKS_NUM),str(chunk)) time.sleep(1) update_work.do_updates(chunk) sub_count = sub_count + 1 if (sub_count >= CHUNKS_NUM): commit_it() sub_count = 0 return 1
def update_keys(keys): global args keys = (k for k in keys if k.count("/") == 2 and k.split("/")[1] in ["books", "authors", "works"]) update_work.clear_monkeypatch_cache(max_size=10000) print str(args) update_work.load_configs(args.ol_url, args.config, 'default') count = 0 for chunk in web.group(keys, 100): chunk = list(chunk) count += len(chunk) update_work.do_updates(chunk) if count: logger.info("updated %d documents", count) return count
def update_keys(keys): global args keys = (k for k in keys if k.count("/") == 2 and k.split("/")[1] in ["books", "authors", "works"]) update_work.clear_monkeypatch_cache(max_size=10000) print str(args) update_work.load_configs(args.ol_url,args.config,'default') count = 0 for chunk in web.group(keys, 100): chunk = list(chunk) count += len(chunk) update_work.do_updates(chunk) # update_work.update_keys(chunk, commit=False) if count: logger.info("updated %d documents", count) return count
def update_keys(keys): if not keys: return 0 # FIXME: Some kind of hack introduced to work around DB connectivity issue global args logger.debug("Args: %s" % str(args)) update_work.load_configs(args.ol_url, args.config, 'default') keys = (k for k in keys if k.count("/") == 2 and k.split("/")[1] in ["books", "authors", "works"]) update_work.clear_monkeypatch_cache(max_size=10000) count = 0 for chunk in web.group(keys, 100): chunk = list(chunk) count += len(chunk) update_work.do_updates(chunk) if count: logger.info("updated %d documents", count) return count