def create_uast_source(args, session_name, select=HeadFiles, language_selector=None, extract_uast=True): if args.parquet: parquet_loader_args = filter_kwargs(args.__dict__, create_parquet_loader) start_point = create_parquet_loader(session_name, **parquet_loader_args) root = start_point if extract_uast and "uast" not in [ col.name for col in start_point.execute().schema ]: raise ValueError("The parquet files do not contain UASTs.") else: engine_args = filter_kwargs(args.__dict__, create_engine) root = create_engine(session_name, **engine_args) if language_selector is None: language_selector = LanguageSelector(languages=args.languages) start_point = Ignition(root, explain=args.explain) \ .link(select()) \ .link(language_selector) if extract_uast: start_point = start_point.link(UastExtractor()) return root, start_point
def create_file_source(args: argparse.Namespace, session_name: str): if args.parquet: parquet_loader_args = filter_kwargs(args.__dict__, create_parquet_loader) root = create_parquet_loader(session_name, **parquet_loader_args) file_source = root.link(LanguageSelector.maybe(languages=args.languages, blacklist=args.blacklist)) else: engine_args = filter_kwargs(args.__dict__, create_engine) root = Ignition(create_engine(session_name, **engine_args), explain=args.explain) file_source = root.link(DzhigurdaFiles(args.dzhigurda)) if args.languages is not None: file_source = file_source \ .link(LanguageExtractor()) \ .link(LanguageSelector(languages=args.languages, blacklist=args.blacklist)) return root, file_source
def merge_coocc_spark(df, filepaths, log, args): session_name = "merge_coocc-%s" % uuid4() session = create_spark(session_name, **filter_kwargs(args.__dict__, create_spark)) spark_context = session.sparkContext global_index = spark_context.broadcast(df.order) coocc_rdds = [] def local_to_global(local_index): """ Converts token index of co-occurrence matrix to the common index. For example index, 5 correspond to `get` token for a current model. And `get` have index 7 in the result. So we convert 5 to `get` via tokens list and `get` to 7 via global_index mapping. If global_index do not have `get` token, it returns -1. """ return global_index.value.get(tokens.value[local_index], -1) for path, coocc in load_and_check(filepaths, log): rdd = coocc.matrix_to_rdd(spark_context) # rdd structure: ((row, col), weight) log.info("Broadcasting tokens order for %s model...", path) tokens = spark_context.broadcast(coocc.tokens) coocc_rdds.append( rdd.map(lambda row: ((local_to_global(row[0][0]), local_to_global(row[0][1])), np.uint32(row[1]))) .filter(lambda row: row[0][0] >= 0)) log.info("Calculating the union of cooccurrence matrices...") rdd = spark_context \ .union(coocc_rdds) \ .reduceByKey(lambda x, y: min(MAX_INT32, x + y)) CooccModelSaver(args.output, df)(rdd)
def evaluate_communities(args): log = logging.getLogger("evalcc") model = CommunitiesModel().load(args.input) configure(args) spark = create_spark("evalcc-%s" % uuid4(), **filter_kwargs(args.__dict__, create_spark)) log.info("Preparing the communities' RDD") items = [] for i, c in progress_bar(enumerate(model.communities), log, expected_size=len(model.communities)): for m in c: if m < len(model.id_to_element): items.append(Row(sha1=model.id_to_element[m], community=i)) log.info("Running") items_in_spark = spark.sparkContext.parallelize(items).toDF() bags = spark \ .read \ .format("org.apache.spark.sql.cassandra") \ .options(table=args.tables["bags"], keyspace=args.keyspace) \ .load() log.info("Loaded the bags, calculating the vocabulary") vocabulary = bags.drop( "sha1", "value").distinct().rdd.map(lambda x: x.item).collect() vocabulary = {v: i for i, v in enumerate(vocabulary)} log.info("Vocabulary size: %d", len(vocabulary)) element_to_id = {e: i for i, e in enumerate(model.id_to_element)} metrics = items_in_spark.join(bags, "sha1").rdd \ .map(lambda r: (r.community, (element_to_id[r.sha1], vocabulary[r.item], r.value))) \ .groupByKey() \ .map(CommunityEvaluator(args.threshold, len(vocabulary))) \ .reduce(lambda v1, v2: [v1[i] + v2[i] for i in range(4)]) log.info("Total misses: %d", metrics[0]) log.info("Average normalized misses: %f", metrics[1] / len(model.communities)) log.info("Total loss: %f", metrics[2]) log.info("Average normalized loss: %f", numpy.sqrt(metrics[3] / len(model.communities)))
def hash_batches(args): log = logging.getLogger("hash") log.info("Loading files from %s", args.input) loader = BOWLoader(args.input) log.info("%d batches", len(loader)) # Check batches if not loader: return htnum, band_size = calc_hashtable_params( args.threshold, args.size, args.false_positive_weight, args.false_negative_weight) log.info("Number of hash tables: %d", htnum) log.info("Band size: %d", band_size) cassandra_utils.configure(args) spark_args = filter_kwargs(args.__dict__, create_spark) spark = create_spark("hash-%s" % uuid4(), **spark_args).sparkContext import libMHCUDA # delayed import which requires CUDA and friends tables = args.tables gen = voc_size = None try: for i, bow in enumerate(loader): if voc_size is None: voc_size = bow.matrix.shape[-1] log.info("Initializing the generator") deferred = os.path.isfile(args.params) gen = libMHCUDA.minhash_cuda_init( voc_size, args.size, seed=args.seed, devices=args.devices, verbosity=args.mhc_verbosity, deferred=deferred) if deferred: model = WeightedMinHashParameters().load(args.params) libMHCUDA.minhash_cuda_assign_vars(gen, model.rs, model.ln_cs, model.betas) else: log.info("Writing %s", args.params) params = libMHCUDA.minhash_cuda_retrieve_vars(gen) WeightedMinHashParameters().construct(*params).save(args.params) if bow.matrix.shape[-1] != voc_size: raise ValueError("The vocabulary sizes do not match: %d != %d" % (bow.matrix.shape[-1], voc_size)) log.info("Processing batch %d / %d", i + 1, len(loader)) # Modify features if needed # TODO(vmarkovtsev): port to the new structure # batches = modify_feature_weights(batches, args) hashes = libMHCUDA.minhash_cuda_calc(gen, bow.matrix) job = [(k, h) for k, h in zip(bow.documents, hashes)] log.info("Saving the hashtables") df = spark.parallelize(job).flatMap(HashExploder(htnum, band_size)).toDF() df.write \ .format("org.apache.spark.sql.cassandra") \ .mode("append") \ .options(table=tables["hashtables"], keyspace=args.keyspace) \ .save() df.write \ .format("org.apache.spark.sql.cassandra") \ .mode("append") \ .options(table=tables["hashtables2"], keyspace=args.keyspace) \ .save() log.info("Saving the hashes") spark.parallelize(job) \ .map(lambda x: Row(sha1=x[0], value=bytearray(x[1].data))) \ .toDF() \ .write \ .format("org.apache.spark.sql.cassandra") \ .mode("append") \ .options(table=tables["hashes"], keyspace=args.keyspace) \ .save() finally: libMHCUDA.minhash_cuda_fini(gen)
def hash_batches(args): log = logging.getLogger("hash") log.info("Loading files from %s", args.input) loader = BOWLoader(args.input) log.info("%d batches", len(loader)) # Check batches if not loader: return htnum, band_size = calc_hashtable_params(args.threshold, args.size, args.false_positive_weight, args.false_negative_weight) log.info("Number of hash tables: %d", htnum) log.info("Band size: %d", band_size) cassandra_utils.configure(args) spark_args = filter_kwargs(args.__dict__, create_spark) spark = create_spark("hash-%s" % uuid4(), **spark_args).sparkContext import libMHCUDA # delayed import which requires CUDA and friends tables = args.tables gen = voc_size = None try: for i, bow in enumerate(loader): if voc_size is None: voc_size = bow.matrix.shape[-1] log.info("Initializing the generator") deferred = os.path.isfile(args.params) gen = libMHCUDA.minhash_cuda_init(voc_size, args.size, seed=args.seed, devices=args.devices, verbosity=args.mhc_verbosity, deferred=deferred) if deferred: model = WeightedMinHashParameters().load(args.params) libMHCUDA.minhash_cuda_assign_vars(gen, model.rs, model.ln_cs, model.betas) else: log.info("Writing %s", args.params) params = libMHCUDA.minhash_cuda_retrieve_vars(gen) WeightedMinHashParameters().construct(*params).save( args.params) if bow.matrix.shape[-1] != voc_size: raise ValueError( "The vocabulary sizes do not match: %d != %d" % (bow.matrix.shape[-1], voc_size)) log.info("Processing batch %d / %d", i + 1, len(loader)) # Modify features if needed # TODO(vmarkovtsev): port to the new structure # batches = modify_feature_weights(batches, args) hashes = libMHCUDA.minhash_cuda_calc(gen, bow.matrix) job = [(k, h) for k, h in zip(bow.documents, hashes)] log.info("Saving the hashtables") df = spark.parallelize(job).flatMap(HashExploder( htnum, band_size)).toDF() df.write \ .format("org.apache.spark.sql.cassandra") \ .mode("append") \ .options(table=tables["hashtables"], keyspace=args.keyspace) \ .save() df.write \ .format("org.apache.spark.sql.cassandra") \ .mode("append") \ .options(table=tables["hashtables2"], keyspace=args.keyspace) \ .save() log.info("Saving the hashes") spark.parallelize(job) \ .map(lambda x: Row(sha1=x[0], value=bytearray(x[1].data))) \ .toDF() \ .write \ .format("org.apache.spark.sql.cassandra") \ .mode("append") \ .options(table=tables["hashes"], keyspace=args.keyspace) \ .save() finally: libMHCUDA.minhash_cuda_fini(gen)
def detect_communities(args): log = logging.getLogger("cmd") ccsmodel = ConnectedComponentsModel().load(args.input) log.info("Building the connected components") ccs = defaultdict(list) for i, c in enumerate(ccsmodel.id_to_cc): ccs[c].append(i) buckmat = ccsmodel.id_to_buckets buckindices = buckmat.indices buckindptr = buckmat.indptr total_nvertices = buckmat.shape[0] linear = args.edges in ("linear", "1") graphs = [] communities = [] if not linear: log.info("Transposing the matrix") buckmat_csc = buckmat.T.tocsr() fat_ccs = [] for vertices in ccs.values(): if len(vertices) == 1: continue if len(vertices) == 2: communities.append(vertices) continue fat_ccs.append(vertices) log.info("Building %d graphs", len(fat_ccs)) for vertices in progress_bar(fat_ccs, log, expected_size=len(fat_ccs)): if linear: edges = [] weights = [] bucket_weights = buckmat.sum(axis=0) buckets = set() for i in vertices: for j in range(buckindptr[i], buckindptr[i + 1]): bucket = buckindices[j] weights.append(bucket_weights[0, bucket]) bucket += total_nvertices buckets.add(bucket) edges.append((str(i), str(bucket))) else: edges = set() weights = None buckets = set() for i in vertices: for j in range(buckindptr[i], buckindptr[i + 1]): buckets.add(buckindices[j]) for bucket in buckets: buckverts = \ buckmat_csc.indices[buckmat_csc.indptr[bucket]:buckmat_csc.indptr[bucket + 1]] for i, x in enumerate(buckverts): for y in buckverts: if x < y: edges.add((str(x), str(y))) buckets.clear() edges = list(edges) graph = Graph(directed=False) graph.add_vertices(list(map(str, vertices + list(buckets)))) graph.add_edges(edges) graph.edge_weights = weights graphs.append(graph) log.info("Launching the community detection") detector = CommunityDetector(algorithm=args.algorithm, config=args.params) if not args.no_spark: spark = create_spark("cmd-%s" % uuid4(), **filter_kwargs(args.__dict__, create_spark)).sparkContext communities.extend( spark.parallelize(graphs).flatMap(detector).collect()) else: communities.extend( chain.from_iterable( progress_bar((detector(g) for g in graphs), log, expected_size=len(graphs)))) log.info("Overall communities: %d", len(communities)) log.info("Average community size: %.1f", numpy.mean([len(c) for c in communities])) log.info("Median community size: %.1f", numpy.median([len(c) for c in communities])) log.info("Max community size: %d", max(map(len, communities))) log.info("Writing %s", args.output) CommunitiesModel().construct(communities, ccsmodel.id_to_element).save(args.output)
def warmup(args): engine_args = filter_kwargs(args.__dict__, create_engine) create_engine("warmup", "/tmp", **engine_args)