def main():
results = {}
miter = MongoDocumentIterator(fields=['JarMetadata'])
print 'Found %d Documents' % (miter.total(),)
while miter.has_next():
d = miter.next()
try:
if d is not None:
print 'Working %d of %d' % (miter.count(), miter.total(),)
node_key = '%s||%s||%s' % (d['JarMetadata']['group_id'], d['JarMetadata']['artifact_id'], d['JarMetadata']['version'])
deps = []
for dep in d.get('JarMetadata', {}).get('dependencies', {}):
if isinstance(dep, dict):
dep_group_id = dep.get('groupId', None)
dep_artifact_id = dep.get('artifactId', None)
dep_version = dep.get('version', None)
if dep_group_id is None or dep_artifact_id is None:
continue
if dep_version is None:
deps.append('%s||%s' % (dep_group_id, dep_artifact_id))
else:
deps.append('%s||%s||%s' % (dep_group_id, dep_artifact_id, dep_version))
results[node_key] = {'dependencies ' : deps, 'timestamp' : d['JarMetadata'].get('jar_last_modification_date', 0), 'version_order' : d['JarMetadata']['version_order']}
except Exception, e:
print d
def main():
versions = []
dup_versions = ArrayCount()
miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version'])
print 'Found %d Documents' % (miter.total(),)
while miter.has_next():
d = miter.next()
if d is not None:
group_id = d['JarMetadata']['group_id']
artifact_id = d['JarMetadata']['artifact_id']
version = d['JarMetadata']['version']
ga = '%s||%s||%s' % (group_id, artifact_id, version)
if ga not in versions:
versions.append(ga)
else:
dup_versions.incr(ga)
print '[%d:%d:%d]: Processed %s' % (dup_versions.item_count(), len(versions), miter.count(), ga)
print 'Total documents: %d, dups: %d, versions: %d' % (miter.total(), dup_versions.item_count(), len(versions))
save_to_file('duplicates.json', json.dumps(dup_versions.get_series()))
def main():
results = {}
miter = MongoDocumentIterator(fields=['JarMetadata'])
print 'Found %d Documents' % (miter.total(), )
while miter.has_next():
d = miter.next()
try:
if d is not None:
print 'Working %d of %d' % (
miter.count(),
miter.total(),
)
node_key = '%s||%s||%s' % (d['JarMetadata']['group_id'],
d['JarMetadata']['artifact_id'],
d['JarMetadata']['version'])
deps = []
for dep in d.get('JarMetadata', {}).get('dependencies', {}):
if isinstance(dep, dict):
dep_group_id = dep.get('groupId', None)
dep_artifact_id = dep.get('artifactId', None)
dep_version = dep.get('version', None)
if dep_group_id is None or dep_artifact_id is None:
continue
if dep_version is None:
deps.append('%s||%s' %
(dep_group_id, dep_artifact_id))
else:
deps.append(
'%s||%s||%s' %
(dep_group_id, dep_artifact_id, dep_version))
results[node_key] = {
'dependencies ':
deps,
'timestamp':
d['JarMetadata'].get('jar_last_modification_date', 0),
'version_order':
d['JarMetadata']['version_order']
}
except Exception, e:
print d
def main():
results = ArrayCount()
miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id'])
print 'Found %d Documents' % (miter.total(),)
while miter.has_next():
d = miter.next()
if d is not None:
group_id = d['JarMetadata']['group_id']
artifact_id = d['JarMetadata']['artifact_id']
ga = '%s||%s' % (group_id, artifact_id)
results.incr(ga)
print 'Working %d of %d' % (miter.count(), miter.total(),)
save_to_file('project_versions.json', json.dumps(results.get_series()))
def main():
project_list = []
miter = MongoDocumentIterator(query={'BugCollection.BugInstance.category':'SECURITY'},fields=['JarMetadata.group_id', 'JarMetadata.artifact_id'])
print 'Found %d documents with SECURITY bugs' % (miter.total(),)
while miter.has_next():
d = miter.next()
if d is not None:
print '%d of %d (security)' % (miter.count(), miter.total())
group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet')
artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet')
project_key = '%s||%s' % (group_id, artifact_id)
if project_key not in project_list:
project_list.append(project_key)
miter = MongoDocumentIterator(query={'BugCollection.BugInstance.category':'MALICIOUS_CODE'},fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version','BugCollection.BugInstance.category', 'BugCollection.BugInstance.type'])
print 'Found %d documents with MALICIOUS_CODE bugs' % (miter.total(),)
while miter.has_next():
d = miter.next()
if d is not None:
print '%d of %d (malicious_code)' % (miter.count(), miter.total())
group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet')
artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet')
project_key = '%s||%s' % (group_id, artifact_id)
if project_key not in project_list:
project_list.append(project_key)
print "Total: %d Projects" % (len(project_list),)
save_to_file('vuln_projects.json', json.dumps(project_list))
def main():
project_list = []
miter = MongoDocumentIterator(
query={'BugCollection.BugInstance.category': 'SECURITY'},
fields=['JarMetadata.group_id', 'JarMetadata.artifact_id'])
print 'Found %d documents with SECURITY bugs' % (miter.total(), )
while miter.has_next():
d = miter.next()
if d is not None:
print '%d of %d (security)' % (miter.count(), miter.total())
group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet')
artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet')
project_key = '%s||%s' % (group_id, artifact_id)
if project_key not in project_list:
project_list.append(project_key)
miter = MongoDocumentIterator(
query={'BugCollection.BugInstance.category': 'MALICIOUS_CODE'},
fields=[
'JarMetadata.group_id', 'JarMetadata.artifact_id',
'JarMetadata.version', 'BugCollection.BugInstance.category',
'BugCollection.BugInstance.type'
])
print 'Found %d documents with MALICIOUS_CODE bugs' % (miter.total(), )
while miter.has_next():
d = miter.next()
if d is not None:
print '%d of %d (malicious_code)' % (miter.count(), miter.total())
group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet')
artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet')
project_key = '%s||%s' % (group_id, artifact_id)
if project_key not in project_list:
project_list.append(project_key)
print "Total: %d Projects" % (len(project_list), )
save_to_file('vuln_projects.json', json.dumps(project_list))
