def _process_caption_data(phase, ann_file=None, max_length=None):
if phase in ['val', 'train']:
caption_data = load_json(ann_file)
if phase == 'val':
caption_data['type'] = 'caption'
# id_to_filename is a dictionary such as {image_id: filename]}
id_to_filename = {
image['id']: image['file_name']
for image in caption_data['images']
}
# data is a list of dictionary which contains 'captions', 'file_name' and 'image_id' as key.
for i, annotation in enumerate(caption_data['annotations']):
image_id = annotation['image_id']
caption_data['annotations'][i]['file_name'] = id_to_filename[
image_id]
if phase == 'train':
del_idx = []
for i, annotation in enumerate(caption_data['annotations']):
caption = annotation['caption']
caption = caption.replace('.', '').replace(',', '').replace(
"'", '').replace('"', '')
caption = caption.replace('&', 'and').replace('(', '').replace(
')', '').replace('-', ' ')
caption = ' '.join(caption.split()) # replace multiple spaces
caption_data['annotations'][i]['caption'] = caption.lower()
if max_length != None and len(caption.split(' ')) > max_length:
del_idx.append(i)
# delete captions if size is larger than max_length
print("The number of captions before deletion: %d" %
len(caption_data['annotations']))
for idx in sorted(del_idx, reverse=True):
del caption_data['annotations'][idx]
print("The number of captions after deletion: %d" %
len(caption_data['annotations']))
save_json(caption_data,
os.path.join('data', phase,
ann_file.split('/')[-1]))
def __init_service_file_mp(self):
if os.path.exists(self.SERVICE_FILE_PATH):
utils.write_log("Load service path from cache: %s",
self.SERVICE_FILE_PATH)
self.service_file_mp = utils.load_json(self.SERVICE_FILE_PATH)
return
sr_cls = osp.join(CONFIG.DEFAULT_AOSP_PATH,
CONFIG.SERVICE_REGISTER_CLS)
service_manager = get_service_registers(sr_cls)
from core.JniAPIExtractor import SourceFileScanner
jni = SourceFileScanner(CONFIG.DEFAULT_AOSP_PATH)
java_files = jni.get_files(".java")
for jf in java_files:
for k, it in service_manager.items():
cls = it["cls"] + ".java"
fi = jf.split(os.sep).pop()
if fi == cls:
self.service_file_mp[it["cls"]] = jf
utils.write_json(self.service_file_mp, self.SERVICE_FILE_PATH)
def main():
args = parser.parse_args()
# phases to be processed.
phases = [phase.strip() for phase in args.phases.split(',')]
# annotation files to be processed
if sorted(phases) == sorted(['train', 'val', 'test'
]) and args.ann_files == '':
tmplt = 'data/annotations/captions_%s2017.json'
ann_files = [tmplt % 'train', tmplt % 'val', '']
else:
ann_files = [
ann_file.strip() for ann_file in args.ann_files.split(',')
]
# batch size for extracting feature vectors.
batch_size = args.batch_size
# maximum length of caption(number of word). if caption is longer than max_length, deleted.
max_length = args.max_length
# if word occurs less than word_count_threshold in training dataset, the word index is special unknown token.
word_count_threshold = args.word_count_threshold
vocab_size = args.vocab_size
for phase, ann_file in zip(phases, ann_files):
_process_caption_data(phase, ann_file=ann_file, max_length=max_length)
if phase == 'train':
captions_data = load_json('./data/train/captions_train2017.json')
word_to_idx = _build_vocab(captions_data,
threshold=word_count_threshold,
vocab_size=vocab_size)
save_json(word_to_idx, './data/word_to_idx.json')
new_captions_data = _build_caption_vector(captions_data,
word_to_idx=word_to_idx,
max_length=max_length)
save_json(new_captions_data, ann_file)
print('Finished processing caption data')
feature_extractor = FeatureExtractor(model_name='resnet101', layer=3)
for phase in phases:
if not os.path.isdir('./data/%s/feats/' % phase):
os.makedirs('./data/%s/feats/' % phase)
image_paths = os.listdir('./image/%s/' % phase)
dataset = CocoImageDataset(root='./image/%s/' % phase,
image_paths=image_paths)
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
num_workers=8)
for batch_paths, batch_images in tqdm(data_loader):
feats = feature_extractor(batch_images).data.cpu().numpy()
feats = feats.reshape(-1, feats.shape[1] * feats.shape[2],
feats.shape[-1])
for j in range(len(feats)):
np.save('./data/%s/feats/%s.npy' % (phase, batch_paths[j]),
feats[j])
import sys
import time
from core.requester import requester
from core.utils import host, load_json
details = load_json(sys.path[0] + '/db/details.json')
def passive_tests(url, headers):
root = host(url)
acao_header, acac_header = headers.get('access-control-allow-origin', None), headers.get('access-control-allow-credentials', None)
if acao_header == '*':
info = details['wildcard value']
info['acao header'] = acao_header
info['acac header'] = acac_header
return {url : info}
if root:
if host(acao_header) and root != host(acao_header):
info = details['third party allowed']
info['acao header'] = acao_header
info['acac header'] = acac_header
return {url : info}
def active_tests(url, query, root, scheme, header_dict, delay):
origin = scheme + '://' + root
headers = requester(url, scheme, query, header_dict, origin)
if headers:
origin = root + '://' + 'example.com'
acao_header, acac_header = headers.get('access-control-allow-origin', None), headers.get('access-control-allow-credentials', None)
if acao_header and acao_header == (origin):
hostfile = '-iL ' + input_file
else:
host = ' %s ' % host
use_cpe = True
if method == 'software':
use_cpe = False
ports_to_scan = '0-65535'
if quick:
ports_to_scan = ','.join(core.memory.config['top_ports'])
database = {}
print('%s Deploying masscan' % run)
masscan_saved = load_json(savefile)
if not masscan_saved:
file = open(savefile, 'w+')
file.close()
exclude = [host for host in masscan_saved]
if exclude:
exclude = ' --exclude ' + ','.join(exclude) + ' '
else:
exclude = ''
os.system('masscan%s-p%s --rate %i -oG %s %s %s >/dev/null 2>&1' % (host, ports_to_scan, args.rate, savefile, hostfile, exclude))
master_db = parse_masscan(savefile)
for host in masscan_saved:
master_db[host] = masscan_saved[host]
write_json(savefile, master_db)
def cors(target, delay, scheme=False):
url = target
if not target.startswith(('http://', 'https://')):
url = scheme + '://' + url
root = host(url)
parsed = urlparse(url)
netloc = parsed.netloc
scheme = parsed.scheme
url = scheme + '://' + netloc
active = active_tests(url, root, scheme, delay)
return active
details = load_json('./db/details.json')
if target_url:
if target_url.startswith(('http://', 'https://')):
result = cors(target_url, delay)
if result:
print('%s Misconfiguration found!' % good)
print('%s Title: %s' % (info, result))
print('%s Description: %s' %
(info, details[result.lower()]['Description']))
print('%s Severity: %s' %
(info, details[result.lower()]['Severity']))
print('%s Exploitation: %s' %
(info, details[result.lower()]['Exploitation']))
else:
print('%s No misconfiguration found.' % bad)
from bs4 import BeautifulSoup
from core.utils import enable_requests_logging
from core.utils import get_lists_of_values
from core.utils import get_random_user_agent
from core.utils import get_values_multigraph
from core.utils import init_base_parameters
from core.utils import init_multigraph
from core.utils import load_json
logger = logging.getLogger('znpp-node')
ROOT_PATH = os.path.dirname(os.path.realpath(__file__))
ABSOLUTE_PATH = ROOT_PATH + '/data/znpp/'
AIR_TEMPERATURE = load_json(ABSOLUTE_PATH + 'air_temperature.json')
ATM = load_json(ABSOLUTE_PATH + 'atm.json')
HUMIDITY = load_json(ABSOLUTE_PATH + 'humidity.json')
LOADS_UNITS = load_json(ABSOLUTE_PATH + 'loads_units.json')
RADIOLOGY_30KM = load_json(ABSOLUTE_PATH + 'radiology_30km.json')
RADIOLOGY = load_json(ABSOLUTE_PATH + 'radiology.json')
WIND_SPEED = load_json(ABSOLUTE_PATH + 'wind_speed.json')
COLORS = load_json(ROOT_PATH + '/data/colors.json')
def display_config():
# Air temperature
init_multigraph(AIR_TEMPERATURE)
print(('graph_args --base 1000 --upper-limit 20 --lower-limit -20 '
'HRULE:0#a1a1a1'))
init_base_parameters(AIR_TEMPERATURE, COLORS)
import sys
import time
from core.requester import requester
from core.utils import host, load_json
details = load_json(sys.path[0] + "/db/details.json")
def passive_tests(url, headers):
results = []
root = host(url)
acao_header, acac_header = (
headers["access-control-allow-origin"],
headers.get("access-control-allow-credentials", None),
)
if acao_header == "*":
info = details["wildcard value"]
info["acao header"] = acao_header
info["acac header"] = acac_header
results.append({url: info})
if root:
if host(acao_header) and root != host(acao_header):
info = details["third party allowed"]
info["acao header"] = acao_header
info["acac header"] = acac_header
results.append({url: info})
def active_tests(url, root, scheme, header_dict, delay):
results = []
import requests
import sys
from core.utils import enable_requests_logging
from core.utils import get_random_user_agent
from core.utils import get_values_multigraph
from core.utils import init_base_parameters
from core.utils import init_multigraph
from core.utils import load_json
logger = logging.getLogger('rnpp-node')
ROOT_PATH = os.path.dirname(os.path.realpath(__file__))
ABSOLUTE_PATH = ROOT_PATH + '/data/rnpp/'
AIR_TEMPERATURE = load_json(ABSOLUTE_PATH + 'air_temperature.json')
HUMIDITY = load_json(ABSOLUTE_PATH + 'humidity.json')
ATM = load_json(ABSOLUTE_PATH + 'atm.json')
RAINFALL_INTENSITY = load_json(ABSOLUTE_PATH + 'rainfall_intensity.json')
WIND_SPEED = load_json(ABSOLUTE_PATH + 'wind_speed.json')
RADIOLOGY = load_json(ABSOLUTE_PATH + 'radiology.json')
PRODUCTION_ELECTRICITY = load_json(ABSOLUTE_PATH +
'production_electricity.json')
COLORS = load_json(ROOT_PATH + '/data/colors.json')
def display_config():
# Air temperature
init_multigraph(AIR_TEMPERATURE)
print(('graph_args --base 1000 --upper-limit 20 --lower-limit -20 '
'HRULE:0#a1a1a1'))
elif input_file:
target_name = input_file.split('/')[-1]
else:
quit('%s No hosts to scan.' % bad)
if args.outname:
target_name = args.outname.split('/')[-1].split('.')[0]
savefile = args.outname if args.outname else cwd + '/result-' + target_name + '.json'
nmapfile = cwd + '/nmap-' + target_name + '.xml'
if input_file:
print('%s Resolving hostnames to IPs for masscan' % run)
targets = resolver(input_file)
cached_db = load_json(savefile)
if args.use_shodan:
result = shodan(targets, cached_db)
write_json(savefile, result)
print('%s Output saved to %s' % (info, savefile))
quit()
arg_dict = vars(args)
for key in arg_dict:
core.memory.global_vars[key] = arg_dict[key]
flat_targets = ','.join(targets)
hostfile = '-iL ' + input_file if args.input_file else ''
host = ' %s ' % flat_targets if not args.input_file else ' '
use_cpe = True if method == 'software' else False
import time
from core.requester import requester
from core.utils import write_json, load_json
file = './db/vulners_cache.json'
database = load_json(file)
current_time = int(time.time())
if 'time' not in database or (current_time - database.get('time', 0)) > 86400:
database = {'by_cpe': {}, 'by_version': {}}
database['time'] = current_time
def vulners(software, version, cpe=False):
if software and version:
pass
else:
return False
cached = query_cache(software, version, cpe)
if cached:
if cached == 'vulnerable':
return True
else:
return False
kind = 'software'
if cpe:
kind = 'cpe'
data = '{"software": "%s", "version": "%s", "type" : "%s", "maxVulnerabilities" : %i}' % (
software, version, kind, 1)
response = requester('https://vulners.com/api/v3/burp/software/',
