def __init__(self, port, protocol, subprotocol, debug=False, logger=None):
super(Annotator, self).__init__()
self.eligible_tags = list()
self.debug = debug
self.logger = logger
for tag_ in Annotation.iter():
tag = tag_()
if tag.check_port(port) and tag.check_protocol(protocol) \
and tag.check_subprotocol(subprotocol):
self.eligible_tags.append(tag)
def __init__(self, port, protocol, subprotocol, debug=False, logger=None):
super(Annotator, self).__init__()
self.eligible_tags = list()
self.debug = debug
self.logger = logger
for tag_ in Annotation.iter():
tag = tag_()
if tag.check_port(port) and tag.check_protocol(protocol) \
and tag.check_subprotocol(subprotocol):
self.eligible_tags.append(tag)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-p', '--port', type=uint16, help="Target port")
parser.add_argument('-P', '--protocol', type=zsearch_protocol)
parser.add_argument('-S', '--subprotocol', type=zsearch_subprotocol)
parser.add_argument('-T',
'--destination',
default="full_ipv4",
type=str,
choices=["full_ipv4", "alexa_top1mil"])
parser.add_argument('-s', '--scan-id', required=False, type=non_negative)
# parser.add_argument('-t', '--tags', type=tag_class)
parser.add_argument('-I',
'--incoming',
type=subclass_of(Incoming),
default=None)
parser.add_argument('-D',
'--decoder',
type=subclass_of(Decoder),
default=None)
parser.add_argument('-X',
'--transform',
type=subclass_of(Transform),
default=None)
parser.add_argument('-E',
'--encoder',
type=subclass_of(Encoder),
default=JSONEncoder)
parser.add_argument('-O',
'--outgoing',
type=subclass_of(Outgoing),
default=OutputFile)
parser.add_argument('-i',
'--input-file',
default=sys.stdin,
type=argparse.FileType('r'))
parser.add_argument('-l',
'--log-file',
type=argparse.FileType('w'),
default=sys.stderr)
parser.add_argument('-v',
'--log-level',
type=int,
default=Logger.INFO,
choices=range(0, Logger.TRACE + 1))
parser.add_argument('-m',
'--metadata-file',
type=argparse.FileType('w'),
default=sys.stderr)
parser.add_argument('-d', '--debug', action='store_true')
parser.add_argument('-t', '--tests', action='store_true')
parser.add_argument('--safe-import', action='store_true')
parser.add_argument('--no-safe-tests', action='store_true')
args = parser.parse_args()
Annotation.load_annotations(args.safe_import)
if args.tests:
sys.exit(AnnotationTesting().run(args.no_safe_tests))
if not args.port:
sys.stderr.write("ERROR: port (-p/--port) required\n")
sys.exit(1)
if not args.protocol:
sys.stderr.write("ERROR: protocol (-P/--protocol) required\n")
sys.exit(1)
if not args.subprotocol:
sys.stderr.write("ERROR: subprotocol (-S/--subprotocol) required\n")
sys.exit(1)
if not args.scan_id:
sys.stderr.write("ERROR: scanid (-s/--scan-id) required\n")
sys.exit(1)
metadata = dict()
port = args.port
protocol = args.protocol
subprotocol = args.subprotocol
scan_id = args.scan_id
logger = Logger(args.log_file, log_level=args.log_level)
if args.transform is not None:
transform = args.transform(port, protocol, subprotocol, scan_id)
else:
transform = ZMapTransformer.find_transform(port, protocol, subprotocol,
scan_id)
if args.incoming is not None:
incoming = args.incoming(input_file=args.input_file)
elif transform.incoming is not None:
incoming = transform.incoming(input_file=args.input_file)
else:
incoming = InputFile(input_file=args.input_file)
if args.decoder is not None:
decoder = args.decoder(logger=logger)
elif transform.decoder is not None:
decoder = transform.decoder(logger=logger)
else:
decoder = JSONDecoder(logger=logger)
encoder = args.encoder(port, protocol, subprotocol, scan_id)
outgoing = args.outgoing(output_file=sys.stdout,
logger=logger,
destination=args.destination)
tagger = Annotator(port,
protocol,
subprotocol,
debug=args.debug,
logger=logger)
num_tags = len(tagger.eligible_tags)
logger.info("found %d tags" % num_tags)
metadata['eligible_tags'] = num_tags
transforms = [
decoder,
transform,
tagger,
encoder,
]
s = Stream(incoming, outgoing, transforms=transforms, logger=logger)
start_time = datetime.utcnow()
handled, skipped = s.run()
end_time = datetime.utcnow()
duration = end_time - start_time
logger.info("handled %d records" % handled)
logger.info("skipped %d records" % skipped)
metadata['records_handled'] = handled
metadata['records_skipped'] = skipped
metadata['start_time'] = Logger.rfc_time_from_utc(start_time)
metadata['end_time'] = Logger.rfc_time_from_utc(end_time)
metadata['duration'] = int(duration.total_seconds())
args.metadata_file.write(json.dumps(metadata))
args.metadata_file.write("\n")
args.metadata_file.flush()
def run(self, crash_on_failure):
total = 0
failures = 0
have_tests = 0
print "\n" + "=" * 25 + " ZTag Tests " + "=" * 25
for A in sorted(Annotation.iter(), key=lambda x: x.__name__):
total += 1
annotation = A()
tags_match = True
local_metadata_match = True
global_metadata_match = True
errors = []
if annotation.tests:
have_tests += 1
for k, v in annotation.tests.iteritems():
try:
d = Device.from_name(k).get(annotation.port,
annotation.protocol,
annotation.subprotocol)
except:
errors.append(" - %s uses a non-existent device" % k)
tags_match = local_metadata_match = global_metadata_match = False
if crash_on_failure:
raise
continue
if not d:
errors.append(
" - %s does not have any data defined for the protocol targeted by this annotation"
% k)
continue
input = copy.deepcopy(d)
if "metadata" in input:
del input["metadata"]
metadata = Metadata()
try:
output = annotation.process(input,
metadata) or Metadata()
except:
errors.append(" - %s crashed during execution" % k)
tags_match = local_metadata_match = global_metadata_match = False
if crash_on_failure:
raise
continue
expected_tags = set(v.get("tags", []))
expected_global_metadata = v.get("global_metadata", {})
expected_local_metadata = v.get("local_metadata", {})
if set(output.tags) != set(expected_tags):
errors.append(
" - %s (%s):\n %s (received)\n %s (expected)"
%
(k, "tags", str(output.tags), str(expected_tags)))
tags_match = False
if output.local_metadata.to_dict(
with_description=False) != expected_local_metadata:
errors.append(
" - %s (%s):\n %s (received)\n %s (expected)"
% (k, "local metadata",
str(
output.local_metadata.to_dict(
with_description=False)),
str(expected_local_metadata)))
local_metadata_match = False
if output.global_metadata.to_dict(
with_description=False
) != expected_global_metadata:
errors.append(
" - %s (%s):\n %s (received)\n %s (expected)"
% (k, "global metadata",
str(
output.global_metadata.to_dict(
with_description=False)),
str(expected_global_metadata)))
global_metadata_match = False
if errors:
print "{0}: \033[01;31mfail\033[00m".format(
annotation.__class__.__name__)
for error in errors:
print error
failures += 1
else:
print "{0}: \033[1;92m{1}\033[00m".format(
annotation.__class__.__name__, "success")
else:
print "{0}: \033[1;94mno tests\033[00m".format(
annotation.__class__.__name__)
# print summary information
print "\n" + "=" * 25 + " Summary " + "=" * 25 + "\n"
print "annotation modules loaded: %i/%i" % (
Annotation._annotation_annotations_total -
Annotation._annotation_annotations_fail,
Annotation._annotation_annotations_total)
print "test coverage: %i/%i" % (have_tests, total)
print "tests passing: %i/%i" % (have_tests - failures, have_tests)
print "\n"
return 1 if failures or Annotation._annotation_annotations_fail else 0
def run(self, crash_on_failure):
total = 0
failures = 0
have_tests = 0
print "\n" + "="*25 + " ZTag Tests " + "="*25
for A in sorted(Annotation.iter(), key=lambda x: x.__name__):
total += 1
annotation = A()
tags_match = True
local_metadata_match = True
global_metadata_match = True
errors = []
if annotation.tests:
have_tests += 1
for k, v in annotation.tests.iteritems():
try:
d = Device.from_name(k).get(annotation.port, annotation.protocol, annotation.subprotocol)
except:
errors.append(" - %s uses a non-existent device" % k)
tags_match = local_metadata_match = global_metadata_match = False
if crash_on_failure:
raise
continue
if not d:
errors.append(" - %s does not have any data defined for the protocol targeted by this annotation" % k)
continue
input = copy.deepcopy(d)
if "metadata" in input:
del input["metadata"]
metadata = Metadata()
try:
output = annotation.process(input, metadata) or Metadata()
except:
errors.append(" - %s crashed during execution" % k)
tags_match = local_metadata_match = global_metadata_match = False
if crash_on_failure:
raise
continue
expected_tags = set(v.get("tags", []))
expected_global_metadata = v.get("global_metadata", {})
expected_local_metadata = v.get("local_metadata", {})
if set(output.tags) != set(expected_tags):
errors.append(" - %s (%s): %s != %s (expected)" % (k, "tags", str(output.tags), str(expected_tags)))
tags_match = False
if output.local_metadata.to_dict(with_description=False) != expected_local_metadata:
errors.append(" - %s (%s): %s != %s (expected)" % (k, "local metadata", str(output.local_metadata.to_dict()), str(expected_local_metadata)))
local_metadata_match = False
if output.global_metadata.to_dict(with_description=False) != expected_global_metadata:
errors.append(" - %s (%s): %s != %s (expected)" % (k, "global metadata", str(output.global_metadata.to_dict()), str(expected_global_metadata)))
global_metadata_match = False
if errors:
print "{0}: \033[01;31mfail\033[00m".format(annotation.__class__.__name__)
for error in errors:
print error
failures += 1
else:
print "{0}: \033[1;92m{1}\033[00m".format(annotation.__class__.__name__, "success")
else:
print "{0}: \033[1;94mno tests\033[00m".format(annotation.__class__.__name__)
# print summary information
print "\n" + "="*25 + " Summary " + "="*25 + "\n"
print "annotation modules loaded: %i/%i" % (Annotation._annotation_annotations_total - Annotation._annotation_annotations_fail, Annotation._annotation_annotations_total)
print "test coverage: %i/%i" % (have_tests, total)
print "tests passing: %i/%i" % (have_tests - failures, have_tests)
print "\n"
return 1 if failures or Annotation._annotation_annotations_fail else 0
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-p', '--port', type=uint16,
help="Target port")
parser.add_argument('-P', '--protocol',
type=zsearch_protocol)
parser.add_argument('-S', '--subprotocol',
type=zsearch_subprotocol)
parser.add_argument('-T', '--destination', default="full_ipv4",
type=str, choices=["full_ipv4", "alexa_top1mil"])
parser.add_argument('-s', '--scan-id', required=False, type=non_negative)
# parser.add_argument('-t', '--tags', type=tag_class)
parser.add_argument('-I', '--incoming', type=subclass_of(Incoming),
default=None)
parser.add_argument('-D', '--decoder', type=subclass_of(Decoder),
default=JSONDecoder)
parser.add_argument('-X', '--transform', type=subclass_of(Transform),
default=None)
parser.add_argument('-E', '--encoder', type=subclass_of(Encoder),
default=JSONEncoder)
parser.add_argument('-O', '--outgoing', type=subclass_of(Outgoing),
default=OutputFile)
parser.add_argument('-i', '--input-file', default=sys.stdin,
type=argparse.FileType('r'))
parser.add_argument('-l', '--log-file', type=argparse.FileType('w'),
default=sys.stderr)
parser.add_argument('--updates-file', default=sys.stderr,
type=argparse.FileType('w'))
parser.add_argument('-v', '--log-level', type=int, default=Logger.INFO,
choices=range(0, Logger.TRACE + 1))
parser.add_argument('-m', '--metadata-file', type=argparse.FileType('w'),
default=sys.stderr)
parser.add_argument('--strip-domain-prefix', type=str, default=None)
parser.add_argument('-d', '--debug', action='store_true')
parser.add_argument('-t', '--tests', action='store_true')
parser.add_argument('--safe-import', action='store_true')
parser.add_argument('--no-safe-tests', action='store_true')
args = parser.parse_args()
Annotation.load_annotations(args.safe_import)
if args.tests:
sys.exit(AnnotationTesting().run(args.no_safe_tests))
if not args.port:
sys.stderr.write("ERROR: port (-p/--port) required\n")
sys.exit(1)
if not args.protocol:
proto_string = ", ".join(protocols.Protocol._by_pretty_name.keys())
sys.stderr.write("ERROR: protocol (-P/--protocol) required\n")
sys.stderr.write("Registered Protocols: %s\n" % proto_string)
sys.exit(1)
if not args.subprotocol:
subproto_string = ", ".join(
protocols.Subprotocol._by_pretty_name.keys())
sys.stderr.write("ERROR: subprotocol (-S/--subprotocol) required\n")
sys.stderr.write("Registered SubProtocols: %s\n" % subproto_string)
sys.exit(1)
metadata = dict()
port = args.port
protocol = args.protocol
subprotocol = args.subprotocol
scan_id = args.scan_id or 0
transform_kwargs = dict()
transform_args = list()
logger = Logger(args.log_file, log_level=args.log_level)
if args.strip_domain_prefix:
if not args.strip_domain_prefix.endswith("."):
args.strip_domain_prefix += "."
logger.info("stripping prefix %s" % args.strip_domain_prefix)
transform_kwargs['strip_domain_prefix'] = args.strip_domain_prefix
if args.transform is not None:
transform = args.transform(port, protocol, subprotocol, scan_id,
*transform_args, **transform_kwargs)
else:
transform = ZMapTransformer.find_transform(port, protocol, subprotocol,
scan_id, *transform_args, **transform_kwargs)
if args.incoming is not None:
incoming = args.incoming(input_file=args.input_file)
elif transform.incoming is not None:
incoming = transform.incoming(input_file=args.input_file)
else:
incoming = InputFile(input_file=args.input_file)
if args.decoder is not None:
decoder = args.decoder(logger=logger)
elif transform.decoder is not None:
decoder = transform.decoder(logger=logger)
else:
decoder = JSONDecoder(logger=logger)
encoder = args.encoder(port, protocol, subprotocol, scan_id)
outgoing = args.outgoing(output_file=sys.stdout, logger=logger,
destination=args.destination)
tagger = Annotator(port, protocol, subprotocol,
debug=args.debug, logger=logger)
num_tags = len(tagger.eligible_tags)
logger.info("found %d tags" % num_tags)
metadata['eligible_tags'] = num_tags
transforms = [
decoder,
transform,
tagger,
encoder,
]
s = Stream(incoming, outgoing, transforms=transforms, logger=logger, updates=args.updates_file)
start_time = datetime.utcnow()
handled, skipped = s.run()
end_time = datetime.utcnow()
duration = end_time - start_time
logger.info("handled %d records" % handled)
logger.info("skipped %d records" % skipped)
metadata['records_handled'] = handled
metadata['records_skipped'] = skipped
metadata['start_time'] = Logger.rfc_time_from_utc(start_time)
metadata['end_time'] = Logger.rfc_time_from_utc(end_time)
metadata['duration'] = int(duration.total_seconds())
args.metadata_file.write(json.dumps(metadata))
args.metadata_file.write("\n")
args.metadata_file.flush()
