def __init__(self): self.queue_settings = SETTINGS.get("queue", {}) self.cache_settings = SETTINGS.get("cache", {}) self.key_expire_time_in_sec = self.cache_settings.get( "key_expire_time_in_sec", 30) self.cache_index = self.cache_settings.get("index", "index") self.cache_separator = self.cache_settings.get("separator", "::") self.prefix_requests = self.queue_settings.get("prefix_requests", "search::") self.prefix_results = self.queue_settings.get("prefix_results", "create::") self.complete_message = self.queue_settings.get( "complete_message", "create::")
def listen(self): print("Initializing local socket") addr = socket.getaddrinfo(SETTINGS.get("tcp_local_bind"), SETTINGS.get("tcp_port"))[0][-1] self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._sock.setblocking(True) self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._sock.bind(addr) # ATTENTION: When on Micropython 1.10 and ESP8266 ``backlog`` # must be >= 2! # See: https://github.com/micropython/micropython/issues/4511 print("Start listening") # self._sock.listen(5) self._sock.listen(SETTINGS.get("tcp_new_unaccepted_connections")) self._inputs = [self._sock]
def get_pretty_logger(name): """create a useful logger""" logger = logging.Logger(name) filename = settings.get("LOGFILE", "/tmp/antifraud.log") logLevel = settings.get("LOGLEVEL", 6) logger.setLevel(logLevel) LOG_FORMAT = u'%(asctime)s [%(levelname)s] [%(filename)s:%(lineno)d]: %(message)s' formatter = logging.Formatter(LOG_FORMAT) filehandler = logging.FileHandler(filename) console = logging.StreamHandler(sys.stdout) filehandler.setFormatter(formatter) console.setFormatter(formatter) logger.addHandler(filehandler) logger.addHandler(console) return logger
def send_multicast(self, message, multicast_group=None, multicast_port=None): if not multicast_group: multicast_group = SETTINGS.get("multicast_group") if not multicast_port: multicast_port = SETTINGS.get("multicast_port") try: message_raw = message.encode("utf-8") except AttributeError: message_raw = str(message).encode("utf-8") print("Sending multicast to '{}:{}': {!r}".format( multicast_group, multicast_port, message_raw)) SOCK_MULTICAST.sendto(message_raw, (multicast_group, multicast_port))
def mul(): if SETTINGS.get('debug',''): print '请先关闭DEBUG' sockets = tornado.netutil.bind_sockets(options.port) tornado.process.fork_processes(options.process_num) server = tornado.httpserver.HTTPServer(application, xheaders=True) server.add_sockets(sockets) tornado.ioloop.IOLoop.instance().start()
def __init__(self): self.queue_settings = SETTINGS.get("queue", {}) self.channel_to_subscribe_and_publish = self.queue_settings.get( "vulnerability_channel", "vulnerabilityServiceChannel") self.message_to_get_vulnerability = self.queue_settings.get( "message_to_get_vulnerability", "getVulnerability") self.complete_get_vulnerability = self.queue_settings.get( "complete_get_vulnerability", "result::")
def __init__(self, handlers=None, default_host='', transforms=None, **settings): super().__init__(handlers=handlers, default_host=default_host, transforms=transforms, **settings) self.sentry_client = AsyncSentryClient(SETTINGS.get('sentry_dsn'))
def add(filename: str): bookmarks = Bookmarks(SETTINGS.get('bookmarks', 'bookmarks.json')) with open(filename, 'r') as f: data = f.read() soup = BeautifulSoup(data, features="html.parser") # Find all links links = soup.find_all('a', recursive=True) # Find category for each link by checking parents for link in links: bookmarks.add_bookmark(link) bookmarks.save()
def job_for_cwe_updater(): LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Start CWE updater job') if check_internet_connection(): parser = make_parser() cwe_handler = CWEHandler() parser.setContentHandler(cwe_handler) source = SETTINGS.get("cwe", {}).get( "source", "http://cwe.mitre.org/data/xml/cwec_v2.8.xml.zip") try: LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Start downloading file') data, response = get_file(getfile=source) if 'error' not in response: LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Start parsing CWE data') parser.parse(data) LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Complete parsing CWE data') for cwe in cwe_handler.cwe: cwe['description_summary'] = cwe[ 'description_summary'].replace("\t\t\t\t\t", " ") item = {'tag': 'cwe', 'state': 'parsed', 'data': cwe} dc.append(item) LOGINFO_IF_ENABLED( SOURCE_MODULE, "[===========================================================================]" ) LOGINFO_IF_ENABLED( SOURCE_MODULE, '[+] CWE update complete at: {}'.format(datetime.utcnow())) LOGINFO_IF_ENABLED( SOURCE_MODULE, "[===========================================================================]" ) return False else: LOGERR_IF_ENABLED( SOURCE_MODULE, '[-] There are some errors in server response: {}'.format( response)) return False except Exception as ex: LOGERR_IF_ENABLED( SOURCE_MODULE, "Got exception during downloading CWE source: {0}".format(ex)) return False else: LOGERR_IF_ENABLED(SOURCE_MODULE, '[-] No internet connection!') return False
def main(): args = argparser().parse_args() bookmarks = Bookmarks(SETTINGS.get('bookmarks', 'bookmarks.json')) if args.add: add(args.add[0]) if args.dump: dump(args.dump[0]) if args.list: for bm in bookmarks.get(): print(bm) if args.clean: bookmarks.clean() bookmarks.save("cleaned.json")
import logging from settings import SETTINGS logging.basicConfig( format='[%(asctime)s] :: %(message)s', level=logging.DEBUG) logger = logging.getLogger(__file__) logger.setLevel(logging.INFO) enable_extra_logging = SETTINGS.get("enable_extra_logging", False) enable_results_logging = SETTINGS.get("enable_results_logging", False) enable_exception_logging = SETTINGS.get("enable_exception_logging", True) def format_source(source_module): for _ in range(0, 40 - len(source_module)): source_module += '-' source_module += '>' return source_module def LOGINFO_IF_ENABLED(source_module, message): if enable_extra_logging: logger.info(format_source(source_module) + message) def LOGWARN_IF_ENABLED(source_module, message): if enable_extra_logging: logger.warning(format_source(source_module) + message)
import os import re import sys import time import subprocess import schedule from datetime import datetime def now(): return datetime.now().strftime("%Y-%m-%d %H:%M:%S") from subprocess import Popen from settings import SETTINGS monitor_runtime = int(SETTINGS.get("monitor_runtime", 1)) monitor_timeout = int(SETTINGS.get("monitor_timeout", 10)) plugins = plugins_names = [] def get_plugin_name(plugin): if isinstance(plugin, str): return plugin.split('/')[-1][:-3] def load_plugins(): plugin_dir = 'plugins' plugin_prefix = 'plg' listdir = os.listdir(os.path.join(os.path.dirname(__file__), plugin_dir)) plugins = []
def run(self): subscriber = queue.pubsub() subscriber.subscribe([self.channel_to_subscribe_and_publish]) for message in subscriber.listen(): data = message.get("data", {}) if data == 1: pass else: if isinstance(data, bytes): data = data.decode("utf-8") if self.message_to_get_vulnerability in data: id_of_request = data[len(self.message_to_get_vulnerability ) + 10:] uniq_id = data[len(self.message_to_get_vulnerability) + 1:len(self.message_to_get_vulnerability) + 9] try: int(id_of_request) except ValueError: search_options = json.loads(id_of_request) new_collection_name = self.complete_get_vulnerability + uniq_id + ':' + search_options[ 'name'] + ':' + search_options['version'] connect_database() if search_options['version'] == '*': count = VULNERABILITIES.select().where( VULNERABILITIES.component == search_options['name']).count() vulnerability = list( VULNERABILITIES.select().where( (VULNERABILITIES.component == search_options['name']) & (VULNERABILITIES.cvss >= search_options['sort'])).offset( search_options['skip']).limit( search_options['limit']).order_by( VULNERABILITIES.cvss.desc())) else: count = VULNERABILITIES.select().where( (VULNERABILITIES.component == search_options['name']) & (VULNERABILITIES.version == search_options['version'])).count() vulnerability = list( VULNERABILITIES.select().where( (VULNERABILITIES.component == search_options['name']) & (VULNERABILITIES.version == search_options['version']) & (VULNERABILITIES.cvss >= search_options['sort'])).offset( search_options['skip']).limit( search_options['limit']).order_by( VULNERABILITIES.cvss.desc())) queue.rpush(new_collection_name, count) for i in vulnerability: queue.rpush( new_collection_name, serialize_as_json__for_cache( reformat_vulner_for_output(i.to_json))) disconnect_database() queue.publish( channel=self.channel_to_subscribe_and_publish, message=new_collection_name) except Exception as ex: if SETTINGS.get("debug", False): print(ex) else: new_collection_name = self.complete_get_vulnerability + uniq_id + ':' + id_of_request connect_database() vulnerability = list(VULNERABILITIES.select().where( VULNERABILITIES.id == id_of_request)) vlist = [] for i in vulnerability: vlist.append(i.to_json) if len(vlist) == 1: queue.rpush( new_collection_name, serialize_as_json__for_cache( reformat_vulner_for_output(vlist[0]))) disconnect_database() queue.publish( channel=self.channel_to_subscribe_and_publish, message=new_collection_name)
def dump(filename: str): bookmarks = Bookmarks(SETTINGS.get('bookmarks', 'bookmarks.json')) bookmarks.dump(filename)
import peewee import logging import zipfile import bz2 import gzip import urllib.request as req from settings import SETTINGS from io import BytesIO from datetime import datetime from model_thn import THN logging.basicConfig(format='%(name)s >> [%(asctime)s] :: %(message)s', level=logging.DEBUG) logger = logging.getLogger(__file__) logger.setLevel(logging.INFO) debug = bool(SETTINGS.get("debug", True)) enable_extra_logging = SETTINGS.get("enable_extra_logging", False) enable_results_logging = SETTINGS.get("enable_results_logging", False) enable_exception_logging = SETTINGS.get("enable_exception_logging", True) drop_thn_table_before = SETTINGS.get("drop_thn_table_before", False) undefined = SETTINGS.get("undefined", "undefined") POSTGRES = SETTINGS.get("postgres", {}) pg_default_database = POSTGRES.get("database", "updater_db") pg_default_user = POSTGRES.get("user", "admin") pg_default_password = POSTGRES.get("password", "123") pg_default_host = POSTGRES.get("host", "localhost") pg_default_port = POSTGRES.get("port", "5432")
import json import os import sqlite3 import mimetypes from flask import Flask, request, flash, make_response, g from flask_cors import CORS from file_domain import File from settings import SETTINGS app = Flask(__name__) app.config['UPLOAD_FOLDER'] = SETTINGS.get('UPLOAD_FOLDER', '/tmp') CORS(app) DATABASE = SETTINGS.get('DATABASE') mimetypes.init() def get_db(): db = getattr(g, '_database', None) if db is None: db = g._database = sqlite3.connect(DATABASE) return db # helper to close @app.teardown_appcontext def close_connection(exception): db = getattr(g, '_database', None) if db is not None:
import os import re import peewee from playhouse.postgres_ext import ArrayField from datetime import datetime from settings import SETTINGS POSTGRES = SETTINGS.get("postgres", {}) pg_default_database = POSTGRES.get("database", "updater_db") pg_default_user = POSTGRES.get("user", "admin") pg_default_password = POSTGRES.get("password", "123") pg_default_host = POSTGRES.get("host", "localhost") pg_default_port = POSTGRES.get("port", "5432") pg_drop_before = bool(POSTGRES.get("drop_pg_before", True)) pg_database = os.environ.get("PG_DATABASE", pg_default_database) pg_user = os.environ.get("PG_USER", pg_default_user) pg_password = os.environ.get("PG_PASS", pg_default_password) pg_host = os.environ.get("PG_HOST", pg_default_host) pg_port = os.environ.get("PG_PORT", pg_default_port) database = peewee.PostgresqlDatabase( database=pg_database, user=pg_user, password=pg_password, host=pg_host, port=pg_port )
import logging import requests from dateutil import parser from lxml import html from lxml.cssselect import CSSSelector from datetime import datetime from settings import SETTINGS from model_snyk import SNYK logging.basicConfig(format='%(name)s >> [%(asctime)s] :: %(message)s', level=logging.DEBUG) logger = logging.getLogger(__file__) logger.setLevel(logging.INFO) debug = bool(SETTINGS.get("debug", True)) json_filename = SETTINGS.get("json_filename", "snyk.json") enable_extra_logging = SETTINGS.get("enable_extra_logging", False) enable_results_logging = SETTINGS.get("enable_results_logging", False) enable_exception_logging = SETTINGS.get("enable_exception_logging", True) POSTGRES = SETTINGS.get("postgres", {}) pg_default_database = POSTGRES.get("database", "updater_db") pg_default_user = POSTGRES.get("user", "admin") pg_default_password = POSTGRES.get("password", "123") pg_default_host = POSTGRES.get("host", "localhost") pg_default_port = POSTGRES.get("port", "5432")
def main(): scanner_name = SETTINGS.get('scanner', 'dsseries:usb:0x04F9:0x60E0') print("Sane init") sane.init() """ print("Getting devices") devices = sane.get_devices(localOnly=True) print("Found devices") for device in devices: print(device) print(dir(device)) """ print("Opening scanner") scanner = sane.open(scanner_name) scanner.mode = 'Color' """ print("Scanner options") print(scanner.opt) print("") """ """ Options - mode index: 2 'LineArt', 'Gray', 'Color' - resolution index: 3 75 - 600 - preview index: 4 0, 1 (bool) - brightness cur: 0.0 index: 14 -100.0, 100.0 - contrast index: 15 cur: 0.0 -100.0, 100.0 """ """ print("Scanner optlist") print(scanner.optlist) print("") print("Scanner area") print(scanner.area) print(scanner.get_options()) print(scanner.get_parameters()) """ while True: try: do_scan(scanner) except KeyboardInterrupt: print("Exiting scanner script") break scanner.close()
import peewee import logging import requests from datetime import datetime from settings import SETTINGS from model_ms import MS logging.basicConfig(format='%(name)s >> [%(asctime)s] :: %(message)s', level=logging.DEBUG) logger = logging.getLogger(__file__) logger.setLevel(logging.INFO) debug = bool(SETTINGS.get("debug", True)) undefined = SETTINGS.get("undefined", "undefined") json_filename = SETTINGS.get("json_filename", "snyk.json") enable_extra_logging = SETTINGS.get("enable_extra_logging", False) enable_results_logging = SETTINGS.get("enable_results_logging", False) enable_exception_logging = SETTINGS.get("enable_exception_logging", True) drop_ms_table_before = SETTINGS.get("drop_ms_table_before", False) POSTGRES = SETTINGS.get("postgres", {}) pg_default_database = POSTGRES.get("database", "updater_db") pg_default_user = POSTGRES.get("user", "admin")
import os import json import redis from utils import get_module_name from settings import SETTINGS from logger import LOGINFO_IF_ENABLED, LOGERR_IF_ENABLED SOURCE_MODULE = '[{0}] :: '.format(get_module_name(__file__)) cache_local_settings = SETTINGS.get("cache", {}) cache_default_host = cache_local_settings.get("host", "localhost") cache_default_port = cache_local_settings.get("port", 6379) cache_default_db = cache_local_settings.get("db", 3) store_local_settings = SETTINGS.get("store", {}) store_default_host = store_local_settings.get("host", "localhost") store_default_port = store_local_settings.get("port", 6379) store_default_db = store_local_settings.get("db", 2) stats_local_settings = SETTINGS.get("stats", {}) stats_default_host = stats_local_settings.get("host", "localhost") stats_default_port = stats_local_settings.get("port", 6379) stats_defailt_db = stats_local_settings.get("db", 1) queue_local_settings = SETTINGS.get("queue", {}) queue_default_host = queue_local_settings.get("host", "localhost") queue_default_port = queue_local_settings.get("port", 6379) queue_defailt_db = queue_local_settings.get("db", 0) cache_host = os.environ.get("REDIS_HOST", cache_default_host)
try: from .message import STOMPMessage, STOMPMessageError from .settings import SETTINGS except ImportError: from message import STOMPMessage, STOMPMessageError from settings import SETTINGS CONN_STATUS_RESET = 0 CONN_STATUS_CONNECTED = 1 CONN_STATUS_DISCONNECTED = 128 CONN_STATUS_CLOSED = 256 MESSAGE_BUFFER_SIZE = 128 RECV_BUFFER_SIZE = SETTINGS.get("tcp_recv_buffer_size") EOM = SETTINGS.get("eom") class STOMPConnection(object): def __init__(self, socket_remote): self._sock_remote = socket_remote self._state = CONN_STATUS_RESET # self._message_buffer = bytearray(b"\x00") * MESSAGE_BUFFER_SIZE self._message_buffer = "" self._frames = [] # client related status self._cl_accept_version = "" self._cl_host = "" self._cl_login = "" self._cl_passcode = ""
try: from .settings import SETTINGS except ImportError: from settings import SETTINGS # Some shortcuts # DANGER: will not get updated when settings were updated online EOL = SETTINGS.get("eol") EOM = SETTINGS.get("eom") ACCEPTED_COMMANDS = [ # CLIENT COMMANDS "CONNECT", "SEND", "SUBSCRIBE", "UNSUBSCRIBE", "BEGIN", "COMMIT", "ABORT", "ACK", "NACK", "DISCONNECT", "CONNECT", "STOMP", # SERVER COMMANDS "CONNECTED", "MESSAGE", "RECEIPT", "ERROR", ]
from collections import Counter import hashlib from requests.sessions import session from settings import SETTINGS import json import requests from loguru import logger logger.add(SETTINGS.get('_logfile')) from datetime import datetime from pony.orm import * # https://editor.ponyorm.com/user/shyft/byoctf/designer # this is probably a bit of an overcomplicated db architecture. # this is because I was learning the relationships and how they worked in pony. # things also changed in the project and I left some things in order to not break stuff. db = Database() class Flag(db.Entity): id = PrimaryKey(int, auto=True) challenges = Set('Challenge') description = Optional(str) solves = Set('Solve') flag = Required(str) value = Required(float) unsolved = Optional(bool, default=True)
import os import peewee from playhouse.postgres_ext import ArrayField from datetime import datetime from settings import SETTINGS POSTGRES = SETTINGS.get("postgres", {}) pg_default_database = POSTGRES.get("database", "updater_db") pg_default_user = POSTGRES.get("user", "admin") pg_default_password = POSTGRES.get("password", "123") pg_default_host = POSTGRES.get("host", "localhost") pg_default_port = POSTGRES.get("port", "5432") pg_drop_before = bool(POSTGRES.get("drop_pg_before", True)) pg_database = os.environ.get("PG_DATABASE", pg_default_database) pg_user = os.environ.get("PG_USER", pg_default_user) pg_password = os.environ.get("PG_PASS", pg_default_password) pg_host = os.environ.get("PG_HOST", pg_default_host) pg_port = os.environ.get("PG_PORT", pg_default_port) database = peewee.PostgresqlDatabase(database=pg_database, user=pg_user, password=pg_password, host=pg_host, port=pg_port) class MS(peewee.Model):