def dumpcmd(args):
log = logging.getLogger("dumpcmd")
model = CommunitiesModel().load(args.input)
log.info("Initializing the sha1 resolver")
communities = BatchedCommunityResolver(model, args.batch, get_db(args), args.tables["meta"])
stream_template(args.template, sys.stdout, communities=communities, model=model,
model_path=os.path.abspath(args.input))
def dumpcmd(args):
log = logging.getLogger("dumpcmd")
model = CommunitiesModel().load(args.input)
log.info("Initializing the sha1 resolver")
communities = BatchedCommunityResolver(model, args.batch, get_db(args), args.tables["meta"])
stream_template(args.template, sys.stdout, communities=communities, model=model,
model_path=os.path.abspath(args.input))
def find_connected_components(args):
log = logging.getLogger("graph")
session = get_db(args)
table = args.tables["hashtables"]
rows = session.execute("SELECT DISTINCT hashtable FROM %s" % table)
hashtables = sorted(r.hashtable for r in rows)
log.info("Detected %d hashtables", len(hashtables))
# Read buckets from database
buckets = []
element_ids = {}
prev_len = 0
for hashtable in hashtables:
rows = session.execute(
"SELECT sha1, value FROM %s WHERE hashtable=%d" % (table, hashtable))
band = None
bucket = []
for row in rows:
eid = element_ids.setdefault(row.sha1, len(element_ids))
if row.value != band:
if band is not None:
buckets.append(bucket.copy())
bucket.clear()
band = row.value
bucket.append(eid)
continue
bucket.append(eid)
if bucket:
buckets.append(bucket)
log.info("Fetched %d, %d buckets", hashtable, len(buckets) - prev_len)
prev_len = len(buckets)
element_to_buckets = [[] for _ in range(len(element_ids))]
for i, bucket in enumerate(buckets):
for element in bucket:
element_to_buckets[element].append(i)
# Statistics about buckets
levels = (logging.ERROR, logging.INFO)
log.info("Number of buckets: %d", len(buckets))
log.log(levels[len(element_ids) >= len(buckets[0])],
"Number of elements: %d", len(element_ids))
epb = sum(map(len, buckets)) / len(buckets)
log.log(levels[epb >= 1], "Average number of elements per bucket: %.1f", epb)
nb = min(map(len, element_to_buckets))
log.log(levels[nb == len(hashtables)], "Min number of buckets per element: %s", nb)
nb = max(map(len, element_to_buckets))
log.log(levels[nb == len(hashtables)], "Max number of buckets per element: %s", nb)
log.info("Running CC analysis")
# Connect components
connected_components_element = _find_connected_component(buckets, element_to_buckets)
log.info("CC number: %d", len(connected_components_element))
log.info("Writing %s", args.output)
ConnectedComponentsModel() \
.construct(connected_components_element, element_to_buckets, element_ids) \
.save(args.output)
def find_connected_components(args):
log = logging.getLogger("graph")
session = get_db(args)
table = args.tables["hashtables"]
rows = session.execute("SELECT DISTINCT hashtable FROM %s" % table)
hashtables = sorted(r.hashtable for r in rows)
log.info("Detected %d hashtables", len(hashtables))
# Read buckets from database
buckets = []
element_ids = {}
prev_len = 0
for hashtable in hashtables:
rows = session.execute(
"SELECT sha1, value FROM %s WHERE hashtable=%d" % (table, hashtable))
band = None
bucket = []
for row in rows:
eid = element_ids.setdefault(row.sha1, len(element_ids))
if row.value != band:
if band is not None:
buckets.append(bucket.copy())
bucket.clear()
band = row.value
bucket.append(eid)
continue
bucket.append(eid)
if bucket:
buckets.append(bucket)
log.info("Fetched %d, %d buckets", hashtable, len(buckets) - prev_len)
prev_len = len(buckets)
element_to_buckets = [[] for _ in range(len(element_ids))]
for i, bucket in enumerate(buckets):
for element in bucket:
element_to_buckets[element].append(i)
# Statistics about buckets
levels = (logging.ERROR, logging.INFO)
log.info("Number of buckets: %d", len(buckets))
log.log(levels[len(element_ids) >= len(buckets[0])],
"Number of elements: %d", len(element_ids))
epb = sum(map(len, buckets)) / len(buckets)
log.log(levels[epb >= 1], "Average number of elements per bucket: %.1f", epb)
nb = min(map(len, element_to_buckets))
log.log(levels[nb == len(hashtables)], "Min number of buckets per element: %s", nb)
nb = max(map(len, element_to_buckets))
log.log(levels[nb == len(hashtables)], "Max number of buckets per element: %s", nb)
log.info("Running CC analysis")
# Connect components
connected_components_element = _find_connected_component(buckets, element_to_buckets)
log.info("CC number: %d", len(connected_components_element))
log.info("Writing %s", args.output)
ConnectedComponentsModel() \
.construct(connected_components_element, element_to_buckets, element_ids) \
.save(args.output)
def query(args):
log = logging.getLogger("query")
session = get_db(args)
tables = args.tables
if args.id:
rows = session.execute(
"SELECT hashtable, value FROM %s WHERE sha1='%s'" %
(tables["hashtables2"], args.id))
bands = [(r.hashtable, r.value) for r in rows]
else:
# args.file
if not args.feature:
log.critical(
"-f / --feature must be specified at least once in file query mode"
)
return 1
if not args.params:
log.critical("-p / --params must be specified in file query mode")
return 1
wmh, bag = hash_file(args)
htnum, band_size = calc_hashtable_params(args.threshold, len(wmh),
args.false_positive_weight,
args.false_negative_weight)
log.info("Number of hash tables: %d", htnum)
log.info("Band size: %d", band_size)
bands = [(i, bytearray(wmh[i * band_size:(i + 1) * band_size].data))
for i in range(htnum)]
similar = set()
log.info("Looking for similar items")
for i, band in bands:
rows = session.execute(
"SELECT sha1 FROM %s WHERE hashtable=%d AND value=0x%s" %
(tables["hashtables"], i, codecs.encode(band, "hex").decode()))
similar.update(r.sha1 for r in rows)
log.info("Fetched %d items", len(similar))
if args.precise:
# Precise bags
vocab = OrderedDocumentFrequencies().load(args.docfreq)
log.info("Calculating the precise result")
if args.id:
rows = session.execute(
"SELECT item, value FROM %s WHERE sha1='%s'" %
(tables["bags"], args.id))
bag = numpy.zeros(len(vocab), dtype=numpy.float32)
for row in rows:
bag[vocab.order[row.item]] = row.value
# Fetch other bags from the DB
precise = []
for x in similar:
rows = session.execute(
"SELECT item, value FROM %s WHERE sha1='%s'" %
(tables["bags"], x))
other_bag = numpy.zeros(len(vocab), dtype=numpy.float32)
for row in rows:
other_bag[vocab.order[row.item]] = row.value
if weighted_jaccard(bag, other_bag) >= args.threshold:
precise.append(x)
log.info("Survived: %.2f", len(precise) / len(similar))
similar = precise
if args.id:
try:
similar.remove(args.id)
except KeyError:
# o_O
pass
similar = [s.split("@")[1] for s in similar]
stream_template(args.template,
sys.stdout,
size=len(similar),
origin=args.id if args.id else os.path.abspath(args.file),
items=BatchedHashResolver(similar, args.batch, session,
tables["meta"]))