def __init__(self):
self.queue_settings = SETTINGS.get("queue", {})
self.cache_settings = SETTINGS.get("cache", {})
self.key_expire_time_in_sec = self.cache_settings.get(
"key_expire_time_in_sec", 30)
self.cache_index = self.cache_settings.get("index", "index")
self.cache_separator = self.cache_settings.get("separator", "::")
self.prefix_requests = self.queue_settings.get("prefix_requests",
"search::")
self.prefix_results = self.queue_settings.get("prefix_results",
"create::")
self.complete_message = self.queue_settings.get(
"complete_message", "create::")
def listen(self):
print("Initializing local socket")
addr = socket.getaddrinfo(SETTINGS.get("tcp_local_bind"),
SETTINGS.get("tcp_port"))[0][-1]
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._sock.setblocking(True)
self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock.bind(addr)
# ATTENTION: When on Micropython 1.10 and ESP8266 ``backlog``
# must be >= 2!
# See: https://github.com/micropython/micropython/issues/4511
print("Start listening")
# self._sock.listen(5)
self._sock.listen(SETTINGS.get("tcp_new_unaccepted_connections"))
self._inputs = [self._sock]
def get_pretty_logger(name):
"""create a useful logger"""
logger = logging.Logger(name)
filename = settings.get("LOGFILE", "/tmp/antifraud.log")
logLevel = settings.get("LOGLEVEL", 6)
logger.setLevel(logLevel)
LOG_FORMAT = u'%(asctime)s [%(levelname)s] [%(filename)s:%(lineno)d]: %(message)s'
formatter = logging.Formatter(LOG_FORMAT)
filehandler = logging.FileHandler(filename)
console = logging.StreamHandler(sys.stdout)
filehandler.setFormatter(formatter)
console.setFormatter(formatter)
logger.addHandler(filehandler)
logger.addHandler(console)
return logger
def send_multicast(self,
message,
multicast_group=None,
multicast_port=None):
if not multicast_group:
multicast_group = SETTINGS.get("multicast_group")
if not multicast_port:
multicast_port = SETTINGS.get("multicast_port")
try:
message_raw = message.encode("utf-8")
except AttributeError:
message_raw = str(message).encode("utf-8")
print("Sending multicast to '{}:{}': {!r}".format(
multicast_group, multicast_port, message_raw))
SOCK_MULTICAST.sendto(message_raw, (multicast_group, multicast_port))
def mul():
if SETTINGS.get('debug',''):
print '请先关闭DEBUG'
sockets = tornado.netutil.bind_sockets(options.port)
tornado.process.fork_processes(options.process_num)
server = tornado.httpserver.HTTPServer(application, xheaders=True)
server.add_sockets(sockets)
tornado.ioloop.IOLoop.instance().start()
def __init__(self):
self.queue_settings = SETTINGS.get("queue", {})
self.channel_to_subscribe_and_publish = self.queue_settings.get(
"vulnerability_channel", "vulnerabilityServiceChannel")
self.message_to_get_vulnerability = self.queue_settings.get(
"message_to_get_vulnerability", "getVulnerability")
self.complete_get_vulnerability = self.queue_settings.get(
"complete_get_vulnerability", "result::")
def __init__(self,
handlers=None,
default_host='',
transforms=None,
**settings):
super().__init__(handlers=handlers,
default_host=default_host,
transforms=transforms,
**settings)
self.sentry_client = AsyncSentryClient(SETTINGS.get('sentry_dsn'))
def add(filename: str):
bookmarks = Bookmarks(SETTINGS.get('bookmarks', 'bookmarks.json'))
with open(filename, 'r') as f:
data = f.read()
soup = BeautifulSoup(data, features="html.parser")
# Find all links
links = soup.find_all('a', recursive=True)
# Find category for each link by checking parents
for link in links:
bookmarks.add_bookmark(link)
bookmarks.save()
def job_for_cwe_updater():
LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Start CWE updater job')
if check_internet_connection():
parser = make_parser()
cwe_handler = CWEHandler()
parser.setContentHandler(cwe_handler)
source = SETTINGS.get("cwe", {}).get(
"source", "http://cwe.mitre.org/data/xml/cwec_v2.8.xml.zip")
try:
LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Start downloading file')
data, response = get_file(getfile=source)
if 'error' not in response:
LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Start parsing CWE data')
parser.parse(data)
LOGINFO_IF_ENABLED(SOURCE_MODULE,
'[+] Complete parsing CWE data')
for cwe in cwe_handler.cwe:
cwe['description_summary'] = cwe[
'description_summary'].replace("\t\t\t\t\t", " ")
item = {'tag': 'cwe', 'state': 'parsed', 'data': cwe}
dc.append(item)
LOGINFO_IF_ENABLED(
SOURCE_MODULE,
"[===========================================================================]"
)
LOGINFO_IF_ENABLED(
SOURCE_MODULE,
'[+] CWE update complete at: {}'.format(datetime.utcnow()))
LOGINFO_IF_ENABLED(
SOURCE_MODULE,
"[===========================================================================]"
)
return False
else:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
'[-] There are some errors in server response: {}'.format(
response))
return False
except Exception as ex:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
"Got exception during downloading CWE source: {0}".format(ex))
return False
else:
LOGERR_IF_ENABLED(SOURCE_MODULE, '[-] No internet connection!')
return False
def main():
args = argparser().parse_args()
bookmarks = Bookmarks(SETTINGS.get('bookmarks', 'bookmarks.json'))
if args.add:
add(args.add[0])
if args.dump:
dump(args.dump[0])
if args.list:
for bm in bookmarks.get():
print(bm)
if args.clean:
bookmarks.clean()
bookmarks.save("cleaned.json")
import logging
from settings import SETTINGS
logging.basicConfig(
format='[%(asctime)s] :: %(message)s',
level=logging.DEBUG)
logger = logging.getLogger(__file__)
logger.setLevel(logging.INFO)
enable_extra_logging = SETTINGS.get("enable_extra_logging", False)
enable_results_logging = SETTINGS.get("enable_results_logging", False)
enable_exception_logging = SETTINGS.get("enable_exception_logging", True)
def format_source(source_module):
for _ in range(0, 40 - len(source_module)):
source_module += '-'
source_module += '>'
return source_module
def LOGINFO_IF_ENABLED(source_module, message):
if enable_extra_logging:
logger.info(format_source(source_module) + message)
def LOGWARN_IF_ENABLED(source_module, message):
if enable_extra_logging:
logger.warning(format_source(source_module) + message)
import os
import re
import sys
import time
import subprocess
import schedule
from datetime import datetime
def now():
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
from subprocess import Popen
from settings import SETTINGS
monitor_runtime = int(SETTINGS.get("monitor_runtime", 1))
monitor_timeout = int(SETTINGS.get("monitor_timeout", 10))
plugins = plugins_names = []
def get_plugin_name(plugin):
if isinstance(plugin, str):
return plugin.split('/')[-1][:-3]
def load_plugins():
plugin_dir = 'plugins'
plugin_prefix = 'plg'
listdir = os.listdir(os.path.join(os.path.dirname(__file__), plugin_dir))
plugins = []
def run(self):
subscriber = queue.pubsub()
subscriber.subscribe([self.channel_to_subscribe_and_publish])
for message in subscriber.listen():
data = message.get("data", {})
if data == 1:
pass
else:
if isinstance(data, bytes):
data = data.decode("utf-8")
if self.message_to_get_vulnerability in data:
id_of_request = data[len(self.message_to_get_vulnerability
) + 10:]
uniq_id = data[len(self.message_to_get_vulnerability) +
1:len(self.message_to_get_vulnerability) +
9]
try:
int(id_of_request)
except ValueError:
search_options = json.loads(id_of_request)
new_collection_name = self.complete_get_vulnerability + uniq_id + ':' + search_options[
'name'] + ':' + search_options['version']
connect_database()
if search_options['version'] == '*':
count = VULNERABILITIES.select().where(
VULNERABILITIES.component ==
search_options['name']).count()
vulnerability = list(
VULNERABILITIES.select().where(
(VULNERABILITIES.component ==
search_options['name']) &
(VULNERABILITIES.cvss >=
search_options['sort'])).offset(
search_options['skip']).limit(
search_options['limit']).order_by(
VULNERABILITIES.cvss.desc()))
else:
count = VULNERABILITIES.select().where(
(VULNERABILITIES.component ==
search_options['name'])
& (VULNERABILITIES.version ==
search_options['version'])).count()
vulnerability = list(
VULNERABILITIES.select().where(
(VULNERABILITIES.component ==
search_options['name'])
& (VULNERABILITIES.version ==
search_options['version']) &
(VULNERABILITIES.cvss >=
search_options['sort'])).offset(
search_options['skip']).limit(
search_options['limit']).order_by(
VULNERABILITIES.cvss.desc()))
queue.rpush(new_collection_name, count)
for i in vulnerability:
queue.rpush(
new_collection_name,
serialize_as_json__for_cache(
reformat_vulner_for_output(i.to_json)))
disconnect_database()
queue.publish(
channel=self.channel_to_subscribe_and_publish,
message=new_collection_name)
except Exception as ex:
if SETTINGS.get("debug", False):
print(ex)
else:
new_collection_name = self.complete_get_vulnerability + uniq_id + ':' + id_of_request
connect_database()
vulnerability = list(VULNERABILITIES.select().where(
VULNERABILITIES.id == id_of_request))
vlist = []
for i in vulnerability:
vlist.append(i.to_json)
if len(vlist) == 1:
queue.rpush(
new_collection_name,
serialize_as_json__for_cache(
reformat_vulner_for_output(vlist[0])))
disconnect_database()
queue.publish(
channel=self.channel_to_subscribe_and_publish,
message=new_collection_name)
def dump(filename: str):
bookmarks = Bookmarks(SETTINGS.get('bookmarks', 'bookmarks.json'))
bookmarks.dump(filename)
import peewee
import logging
import zipfile
import bz2
import gzip
import urllib.request as req
from settings import SETTINGS
from io import BytesIO
from datetime import datetime
from model_thn import THN
logging.basicConfig(format='%(name)s >> [%(asctime)s] :: %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__file__)
logger.setLevel(logging.INFO)
debug = bool(SETTINGS.get("debug", True))
enable_extra_logging = SETTINGS.get("enable_extra_logging", False)
enable_results_logging = SETTINGS.get("enable_results_logging", False)
enable_exception_logging = SETTINGS.get("enable_exception_logging", True)
drop_thn_table_before = SETTINGS.get("drop_thn_table_before", False)
undefined = SETTINGS.get("undefined", "undefined")
POSTGRES = SETTINGS.get("postgres", {})
pg_default_database = POSTGRES.get("database", "updater_db")
pg_default_user = POSTGRES.get("user", "admin")
pg_default_password = POSTGRES.get("password", "123")
pg_default_host = POSTGRES.get("host", "localhost")
pg_default_port = POSTGRES.get("port", "5432")
import json
import os
import sqlite3
import mimetypes
from flask import Flask, request, flash, make_response, g
from flask_cors import CORS
from file_domain import File
from settings import SETTINGS
app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = SETTINGS.get('UPLOAD_FOLDER', '/tmp')
CORS(app)
DATABASE = SETTINGS.get('DATABASE')
mimetypes.init()
def get_db():
db = getattr(g, '_database', None)
if db is None:
db = g._database = sqlite3.connect(DATABASE)
return db
# helper to close
@app.teardown_appcontext
def close_connection(exception):
db = getattr(g, '_database', None)
if db is not None:
import os
import re
import peewee
from playhouse.postgres_ext import ArrayField
from datetime import datetime
from settings import SETTINGS
POSTGRES = SETTINGS.get("postgres", {})
pg_default_database = POSTGRES.get("database", "updater_db")
pg_default_user = POSTGRES.get("user", "admin")
pg_default_password = POSTGRES.get("password", "123")
pg_default_host = POSTGRES.get("host", "localhost")
pg_default_port = POSTGRES.get("port", "5432")
pg_drop_before = bool(POSTGRES.get("drop_pg_before", True))
pg_database = os.environ.get("PG_DATABASE", pg_default_database)
pg_user = os.environ.get("PG_USER", pg_default_user)
pg_password = os.environ.get("PG_PASS", pg_default_password)
pg_host = os.environ.get("PG_HOST", pg_default_host)
pg_port = os.environ.get("PG_PORT", pg_default_port)
database = peewee.PostgresqlDatabase(
database=pg_database,
user=pg_user,
password=pg_password,
host=pg_host,
port=pg_port
)
import logging
import requests
from dateutil import parser
from lxml import html
from lxml.cssselect import CSSSelector
from datetime import datetime
from settings import SETTINGS
from model_snyk import SNYK
logging.basicConfig(format='%(name)s >> [%(asctime)s] :: %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__file__)
logger.setLevel(logging.INFO)
debug = bool(SETTINGS.get("debug", True))
json_filename = SETTINGS.get("json_filename", "snyk.json")
enable_extra_logging = SETTINGS.get("enable_extra_logging", False)
enable_results_logging = SETTINGS.get("enable_results_logging", False)
enable_exception_logging = SETTINGS.get("enable_exception_logging", True)
POSTGRES = SETTINGS.get("postgres", {})
pg_default_database = POSTGRES.get("database", "updater_db")
pg_default_user = POSTGRES.get("user", "admin")
pg_default_password = POSTGRES.get("password", "123")
pg_default_host = POSTGRES.get("host", "localhost")
pg_default_port = POSTGRES.get("port", "5432")
def main():
scanner_name = SETTINGS.get('scanner', 'dsseries:usb:0x04F9:0x60E0')
print("Sane init")
sane.init()
"""
print("Getting devices")
devices = sane.get_devices(localOnly=True)
print("Found devices")
for device in devices:
print(device)
print(dir(device))
"""
print("Opening scanner")
scanner = sane.open(scanner_name)
scanner.mode = 'Color'
"""
print("Scanner options")
print(scanner.opt)
print("")
"""
"""
Options
- mode
index: 2
'LineArt', 'Gray', 'Color'
- resolution
index: 3
75 - 600
- preview
index: 4
0, 1 (bool)
- brightness
cur: 0.0
index: 14
-100.0, 100.0
- contrast
index: 15
cur: 0.0
-100.0, 100.0
"""
"""
print("Scanner optlist")
print(scanner.optlist)
print("")
print("Scanner area")
print(scanner.area)
print(scanner.get_options())
print(scanner.get_parameters())
"""
while True:
try:
do_scan(scanner)
except KeyboardInterrupt:
print("Exiting scanner script")
break
scanner.close()
import peewee
import logging
import requests
from datetime import datetime
from settings import SETTINGS
from model_ms import MS
logging.basicConfig(format='%(name)s >> [%(asctime)s] :: %(message)s',
level=logging.DEBUG)
logger = logging.getLogger(__file__)
logger.setLevel(logging.INFO)
debug = bool(SETTINGS.get("debug", True))
undefined = SETTINGS.get("undefined", "undefined")
json_filename = SETTINGS.get("json_filename", "snyk.json")
enable_extra_logging = SETTINGS.get("enable_extra_logging", False)
enable_results_logging = SETTINGS.get("enable_results_logging", False)
enable_exception_logging = SETTINGS.get("enable_exception_logging", True)
drop_ms_table_before = SETTINGS.get("drop_ms_table_before", False)
POSTGRES = SETTINGS.get("postgres", {})
pg_default_database = POSTGRES.get("database", "updater_db")
pg_default_user = POSTGRES.get("user", "admin")
import os
import json
import redis
from utils import get_module_name
from settings import SETTINGS
from logger import LOGINFO_IF_ENABLED, LOGERR_IF_ENABLED
SOURCE_MODULE = '[{0}] :: '.format(get_module_name(__file__))
cache_local_settings = SETTINGS.get("cache", {})
cache_default_host = cache_local_settings.get("host", "localhost")
cache_default_port = cache_local_settings.get("port", 6379)
cache_default_db = cache_local_settings.get("db", 3)
store_local_settings = SETTINGS.get("store", {})
store_default_host = store_local_settings.get("host", "localhost")
store_default_port = store_local_settings.get("port", 6379)
store_default_db = store_local_settings.get("db", 2)
stats_local_settings = SETTINGS.get("stats", {})
stats_default_host = stats_local_settings.get("host", "localhost")
stats_default_port = stats_local_settings.get("port", 6379)
stats_defailt_db = stats_local_settings.get("db", 1)
queue_local_settings = SETTINGS.get("queue", {})
queue_default_host = queue_local_settings.get("host", "localhost")
queue_default_port = queue_local_settings.get("port", 6379)
queue_defailt_db = queue_local_settings.get("db", 0)
cache_host = os.environ.get("REDIS_HOST", cache_default_host)
try:
from .message import STOMPMessage, STOMPMessageError
from .settings import SETTINGS
except ImportError:
from message import STOMPMessage, STOMPMessageError
from settings import SETTINGS
CONN_STATUS_RESET = 0
CONN_STATUS_CONNECTED = 1
CONN_STATUS_DISCONNECTED = 128
CONN_STATUS_CLOSED = 256
MESSAGE_BUFFER_SIZE = 128
RECV_BUFFER_SIZE = SETTINGS.get("tcp_recv_buffer_size")
EOM = SETTINGS.get("eom")
class STOMPConnection(object):
def __init__(self, socket_remote):
self._sock_remote = socket_remote
self._state = CONN_STATUS_RESET
# self._message_buffer = bytearray(b"\x00") * MESSAGE_BUFFER_SIZE
self._message_buffer = ""
self._frames = []
# client related status
self._cl_accept_version = ""
self._cl_host = ""
self._cl_login = ""
self._cl_passcode = ""
try:
from .settings import SETTINGS
except ImportError:
from settings import SETTINGS
# Some shortcuts
# DANGER: will not get updated when settings were updated online
EOL = SETTINGS.get("eol")
EOM = SETTINGS.get("eom")
ACCEPTED_COMMANDS = [
# CLIENT COMMANDS
"CONNECT",
"SEND",
"SUBSCRIBE",
"UNSUBSCRIBE",
"BEGIN",
"COMMIT",
"ABORT",
"ACK",
"NACK",
"DISCONNECT",
"CONNECT",
"STOMP",
# SERVER COMMANDS
"CONNECTED",
"MESSAGE",
"RECEIPT",
"ERROR",
]
from collections import Counter
import hashlib
from requests.sessions import session
from settings import SETTINGS
import json
import requests
from loguru import logger
logger.add(SETTINGS.get('_logfile'))
from datetime import datetime
from pony.orm import *
# https://editor.ponyorm.com/user/shyft/byoctf/designer
# this is probably a bit of an overcomplicated db architecture.
# this is because I was learning the relationships and how they worked in pony.
# things also changed in the project and I left some things in order to not break stuff.
db = Database()
class Flag(db.Entity):
id = PrimaryKey(int, auto=True)
challenges = Set('Challenge')
description = Optional(str)
solves = Set('Solve')
flag = Required(str)
value = Required(float)
unsolved = Optional(bool, default=True)
import os
import peewee
from playhouse.postgres_ext import ArrayField
from datetime import datetime
from settings import SETTINGS
POSTGRES = SETTINGS.get("postgres", {})
pg_default_database = POSTGRES.get("database", "updater_db")
pg_default_user = POSTGRES.get("user", "admin")
pg_default_password = POSTGRES.get("password", "123")
pg_default_host = POSTGRES.get("host", "localhost")
pg_default_port = POSTGRES.get("port", "5432")
pg_drop_before = bool(POSTGRES.get("drop_pg_before", True))
pg_database = os.environ.get("PG_DATABASE", pg_default_database)
pg_user = os.environ.get("PG_USER", pg_default_user)
pg_password = os.environ.get("PG_PASS", pg_default_password)
pg_host = os.environ.get("PG_HOST", pg_default_host)
pg_port = os.environ.get("PG_PORT", pg_default_port)
database = peewee.PostgresqlDatabase(database=pg_database,
user=pg_user,
password=pg_password,
host=pg_host,
port=pg_port)
class MS(peewee.Model):
