def main(): results = {} miter = MongoDocumentIterator(fields=['JarMetadata']) print 'Found %d Documents' % (miter.total(),) while miter.has_next(): d = miter.next() try: if d is not None: print 'Working %d of %d' % (miter.count(), miter.total(),) node_key = '%s||%s||%s' % (d['JarMetadata']['group_id'], d['JarMetadata']['artifact_id'], d['JarMetadata']['version']) deps = [] for dep in d.get('JarMetadata', {}).get('dependencies', {}): if isinstance(dep, dict): dep_group_id = dep.get('groupId', None) dep_artifact_id = dep.get('artifactId', None) dep_version = dep.get('version', None) if dep_group_id is None or dep_artifact_id is None: continue if dep_version is None: deps.append('%s||%s' % (dep_group_id, dep_artifact_id)) else: deps.append('%s||%s||%s' % (dep_group_id, dep_artifact_id, dep_version)) results[node_key] = {'dependencies ' : deps, 'timestamp' : d['JarMetadata'].get('jar_last_modification_date', 0), 'version_order' : d['JarMetadata']['version_order']} except Exception, e: print d
def main(): versions = [] dup_versions = ArrayCount() miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version']) print 'Found %d Documents' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: group_id = d['JarMetadata']['group_id'] artifact_id = d['JarMetadata']['artifact_id'] version = d['JarMetadata']['version'] ga = '%s||%s||%s' % (group_id, artifact_id, version) if ga not in versions: versions.append(ga) else: dup_versions.incr(ga) print '[%d:%d:%d]: Processed %s' % (dup_versions.item_count(), len(versions), miter.count(), ga) print 'Total documents: %d, dups: %d, versions: %d' % (miter.total(), dup_versions.item_count(), len(versions)) save_to_file('duplicates.json', json.dumps(dup_versions.get_series()))
def main(): project_list = [] miter = MongoDocumentIterator( query={'BugCollection.BugInstance.category': 'SECURITY'}, fields=['JarMetadata.group_id', 'JarMetadata.artifact_id']) print 'Found %d documents with SECURITY bugs' % (miter.total(), ) while miter.has_next(): d = miter.next() if d is not None: print '%d of %d (security)' % (miter.count(), miter.total()) group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet') artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet') project_key = '%s||%s' % (group_id, artifact_id) if project_key not in project_list: project_list.append(project_key) miter = MongoDocumentIterator( query={'BugCollection.BugInstance.category': 'MALICIOUS_CODE'}, fields=[ 'JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type' ]) print 'Found %d documents with MALICIOUS_CODE bugs' % (miter.total(), ) while miter.has_next(): d = miter.next() if d is not None: print '%d of %d (malicious_code)' % (miter.count(), miter.total()) group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet') artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet') project_key = '%s||%s' % (group_id, artifact_id) if project_key not in project_list: project_list.append(project_key) print "Total: %d Projects" % (len(project_list), ) save_to_file('vuln_projects.json', json.dumps(project_list))
def main(): project_list = [] miter = MongoDocumentIterator(query={'BugCollection.BugInstance.category':'SECURITY'},fields=['JarMetadata.group_id', 'JarMetadata.artifact_id']) print 'Found %d documents with SECURITY bugs' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: print '%d of %d (security)' % (miter.count(), miter.total()) group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet') artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet') project_key = '%s||%s' % (group_id, artifact_id) if project_key not in project_list: project_list.append(project_key) miter = MongoDocumentIterator(query={'BugCollection.BugInstance.category':'MALICIOUS_CODE'},fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version','BugCollection.BugInstance.category', 'BugCollection.BugInstance.type']) print 'Found %d documents with MALICIOUS_CODE bugs' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: print '%d of %d (malicious_code)' % (miter.count(), miter.total()) group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet') artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet') project_key = '%s||%s' % (group_id, artifact_id) if project_key not in project_list: project_list.append(project_key) print "Total: %d Projects" % (len(project_list),) save_to_file('vuln_projects.json', json.dumps(project_list))
def main(): results = {} miter = MongoDocumentIterator(fields=['JarMetadata']) print 'Found %d Documents' % (miter.total(), ) while miter.has_next(): d = miter.next() try: if d is not None: print 'Working %d of %d' % ( miter.count(), miter.total(), ) node_key = '%s||%s||%s' % (d['JarMetadata']['group_id'], d['JarMetadata']['artifact_id'], d['JarMetadata']['version']) deps = [] for dep in d.get('JarMetadata', {}).get('dependencies', {}): if isinstance(dep, dict): dep_group_id = dep.get('groupId', None) dep_artifact_id = dep.get('artifactId', None) dep_version = dep.get('version', None) if dep_group_id is None or dep_artifact_id is None: continue if dep_version is None: deps.append('%s||%s' % (dep_group_id, dep_artifact_id)) else: deps.append( '%s||%s||%s' % (dep_group_id, dep_artifact_id, dep_version)) results[node_key] = { 'dependencies ': deps, 'timestamp': d['JarMetadata'].get('jar_last_modification_date', 0), 'version_order': d['JarMetadata']['version_order'] } except Exception, e: print d
def main(): results = ArrayCount() miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id']) print 'Found %d Documents' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: group_id = d['JarMetadata']['group_id'] artifact_id = d['JarMetadata']['artifact_id'] ga = '%s||%s' % (group_id, artifact_id) results.incr(ga) print 'Working %d of %d' % (miter.count(), miter.total(),) save_to_file('project_versions.json', json.dumps(results.get_series()))