def broadcast_links(src_dir): links_list = [] os.path.walk(src_dir,get_objs_in_dir,links_list) b = BroadcastToRedis(config['redis.host'], config['broadcast.queue']) for silo, item in links_list: b.creation(silo, item) return
def get_objs_in_dir(items_list, dirname, fnames): for fname in fnames: a = os.path.join(dirname,fname) if fname == 'obj': item = a.split('pairtree_root')[1].strip('/').split('obj')[0].replace('/', '') silo = a.split('pairtree_root')[0].strip('/').split('/')[-1] if not (silo, item) in items_list: items_list.append((silo, item)) return def broadcast_links(src_dir): links_list = [] os.path.walk(src_dir,get_objs_in_dir,links_list) b = BroadcastToRedis(config['redis.host'], config['broadcast.queue']) for silo, item in links_list: b.creation(silo, item) return src_dirs = [ '/silos', ] for src_dir in src_dirs: print "starting", src_dir links_list = [] os.path.walk(src_dir,get_objs_in_dir,links_list) b = BroadcastToRedis(config['redis.host'], config['broadcast.queue']) for silo, item in links_list: b.creation(silo, item)
s = g.get_rdf_silo(silo_name) itemid = msg.get('id') if itemid and s.exists(itemid): item = s.get_item(itemid) solr_doc = gather_document(silo_name, item) solr.add(_commit=True, **solr_doc) #r.rpush("silochanges", line) #====================================================================== # To add items to redis from rdfdatabank.lib.broadcast import BroadcastToRedis b = BroadcastToRedis("localhost", 'silochanges') b.creation("demo", "Apocalypse-auctm315", ident="admin") b.creation("demo", "Apocalypse-douce249", ident="admin") b.creation("demo", "BibliaPauperum-archgc14", ident="admin") b.creation("demo", "CanticumCanticorum-auctm312", ident="admin") b.creation("demo", "MCSimulation-WW4jet", ident="admin") b.creation("demo", "MCSimulation-WW4jet-CR", ident="admin") b.creation("demo", "MonteCarloSimulations", ident="admin") b.creation("demo", "blockbooks", ident="admin") b.creation("test", "TestSubmission_2", ident="sandbox_user") b.creation("dataflow", "GabrielTest", ident="admin") b.creation("dataflow", "anusha-test", ident="admin") b.creation("dataflow", "anusha-test-testrdf3", ident="admin") b.creation("dataflow", "anusha:test", ident="admin") b.creation("dataflow", "joe-test-2011-09-16-1", ident="admin") b.creation("dataflow", "joetest", ident="admin") b.creation("dataflow", "monica-test", ident="admin")