def source2bags(args):
log = logging.getLogger("bags")
if os.path.exists(args.batches):
log.critical("%s must not exist", args.batches)
return 1
if not args.config:
args.config = []
try:
cassandra_utils.configure(args)
engine = create_engine("source2bags-%s" % uuid4(), **args.__dict__)
extractors = [
__extractors__[s](args.min_docfreq,
**__extractors__[s].get_kwargs_fromcmdline(args))
for s in args.feature
]
pipeline = Engine(engine, explain=args.explain).link(
DzhigurdaFiles(args.dzhigurda))
uasts = pipeline.link(UastExtractor(languages=[args.language]))
if args.persist is not None:
uasts = uasts.link(Cacher(args.persist))
uasts.link(MetadataSaver(args.keyspace, args.tables["meta"]))
uasts = uasts.link(UastDeserializer())
uasts.link(Repo2Quant(extractors, args.nb_partitions))
uasts.link(Repo2DocFreq(extractors))
pipeline.explode()
bags = uasts.link(Repo2WeightedSet(extractors))
if args.persist is not None:
bags = bags.link(Cacher(args.persist))
batcher = bags.link(BagsBatcher(extractors))
batcher.link(BagsBatchSaver(args.batches, batcher))
bags.link(BagsSaver(args.keyspace, args.tables["bags"]))
bags.explode()
log.info("Writing %s", args.docfreq)
batcher.model.save(args.docfreq)
if args.graph:
log.info("Dumping the graph to %s", args.graph)
with open(args.graph, "w") as f:
pipeline.graph(stream=f)
finally:
if args.pause:
input("Press Enter to exit...")
def test_cacher(self):
persistence = SparkDefault.STORAGE_LEVEL
cacher = Cacher(persistence)
cached_data = cacher(self.data)
self.assertTrue(cached_data.is_cached)
self.assertEqual(cacher.persistence, getattr(StorageLevel, persistence))
self.assertIn("head", cacher.__getstate__())
cacher = Cacher.maybe(persistence=None)
uncached_data = cacher(self.data)
self.assertEqual(uncached_data, self.data)
cacher = Cacher.maybe(persistence)
cached_data = cacher(self.data)
self.assertTrue(cached_data.is_cached)
cached_data = Cacher.maybe(persistence)(self.data)
self.assertFalse(cached_data.unpersist().is_cached)