class Listener:
def __init__(
self,
hostname: str = "localhost",
port: int = 8080,
) -> None:
self.hostname = hostname
self.port = port
self.web_server: t.Optional[ThreadingHTTPServer] = None
def start_listening(self):
if self.web_server:
print(
"Listener already started\nTo reset counter call `stop_listening` first."
)
with _Handler.count_lock:
_Handler.post_counter = 0
self.web_server = ThreadingHTTPServer((self.hostname, self.port),
_Handler)
server_thread = threading.Thread(target=self.web_server.serve_forever)
server_thread.daemon = True
server_thread.start()
print(f"Listener server started http://{self.hostname}:{self.port}")
def stop_listening(self):
if self.web_server:
self.web_server.shutdown()
print("Listener server stopped")
self.web_server = None
else:
print("Listener server not found")
def get_post_request_count(self) -> int:
if not self.web_server:
print("Warning: Listener server not found")
return 0
with _Handler.count_lock:
return _Handler.post_counter
def get_submission_latencies(
self,
) -> t.List[t.Tuple[datetime.datetime, datetime.datetime, float]]:
with _Handler.latency_lock:
return _Handler.submission_latencies.copy()
class Harvester(object):
def __init__(self, host='127.0.0.1', port=5000, do_not_track=False):
self.domain_cache: Dict[str, MITMRecord] = {}
self.httpd = ThreadingHTTPServer(
(host, port), ProxyHTTPRequestHandlerWrapper(self.domain_cache, do_not_track))
def serve(self):
try:
self.httpd.serve_forever()
except KeyboardInterrupt:
pass
finally:
self.httpd.shutdown()
def launch_browser(self, browser: Union[browserModule.BrowserEnum, str] = browserModule.BrowserEnum.CHROME,
restart: bool = False, width: int = 400, height: int = 580, args: List[str] = [],
extensions: str = None, verbose: bool = False):
return browserModule.launch(list(self.domain_cache.keys()),
self.httpd.server_address, browser, restart, width, height, args, extensions, verbose)
def get_token_queue(self, domain):
return self.domain_cache[domain].tokens
def _intercept(self, domain: str, sitekey: str, captcha_kind: CaptchaKindEnum, action: str = None):
if not domain_pattern.match(domain):
raise DomainInvalidException(
'You must only give a domain, not a whole URL.')
ret = self.domain_cache[domain] = MITMRecord(
captcha_kind, sitekey, action)
return ret.tokens
def intercept_recaptcha_v2(self, domain: str, sitekey: str):
return self._intercept(domain, sitekey, CaptchaKindEnum.RECAPTCHA_V2, None)
def intercept_recaptcha_v3(self, domain: str, sitekey: str, action: str = None):
return self._intercept(domain, sitekey,
CaptchaKindEnum.RECAPTCHA_V3, action=action)
def intercept_hcaptcha(self, domain: str, sitekey: str):
return self._intercept(domain, sitekey, CaptchaKindEnum.HCAPTCHA, None)
class Client:
def __init__(self):
self.httpd = ThreadingHTTPServer(('localhost', 3000), Handler)
self.connection = RemoteServer(7654)
async def initialize(self):
await self.connection.initialize()
job = Thread(target=self.httpd.serve_forever)
job.start()
async def stop(self):
self.httpd.shutdown()
await self.connection.close()
def send_message(self, message_type: str, details: dict):
data = {'messageType': message_type, 'message': details}
asyncio.create_task(self.connection.send_data(json.dumps(data)))
def poll(self):
return self.connection.poll()
class FontPreviewServer:
def __init__(self, char):
self._char = char
self._families = []
self._server = self._thread = self._port = None
def add_font(self, family):
self._families.append(family)
def start(self):
assert self._server is None
font_previews = []
for family in self._families:
font_previews.append(
__PREVIEW_BLOCK_TEMPLATE__.format(family=family,
char=self._char))
html = __HTML_TEMPLATE__.format(
style=__STYLE__,
font_previews='\n'.join(font_previews),
)
self._server = ThreadingHTTPServer(
('localhost', 0),
functools.partial(FontPreviewRequestHandler,
preview_html_content=html))
self._port = self._server.socket.getsockname()[1]
self._thread = threading.Thread(target=self._server.serve_forever)
self._thread.start()
def stop(self):
assert self._server is not None
self._server.shutdown()
self._thread.join()
self._server = self._thread = self._port = None
@property
def port(self):
return self._port
def stream():
with picamera.PiCamera(resolution='640x480', framerate=24) as camera:
broadcasting = True
frame_buffer = FrameBuffer()
camera.start_recording(frame_buffer, format='h264', profile="baseline")
try:
WebSocketWSGIHandler.http_version = '1.1'
websocketd = make_server(
'',
9000,
server_class=WSGIServer,
handler_class=WebSocketWSGIRequestHandler,
app=WebSocketWSGIApplication(handler_cls=WebSocket))
websocketd.initialize_websockets_manager()
websocketd_thread = Thread(target=websocketd.serve_forever)
httpd = ThreadingHTTPServer(('', 8000), SimpleHTTPRequestHandler)
httpd_thread = Thread(target=httpd.serve_forever)
try:
websocketd_thread.start()
httpd_thread.start()
while broadcasting:
with frame_buffer.condition:
frame_buffer.condition.wait()
websocketd.manager.broadcast(frame_buffer.frame,
binary=True)
except KeyboardInterrupt:
pass
finally:
websocketd.shutdown()
httpd.shutdown()
broadcasting = False
raise KeyboardInterrupt
except KeyboardInterrupt:
pass
finally:
camera.stop_recording()
self.end_headers()
return
try:
results = {}
for header, value in self.headers.items():
# the original introspection endpoint sent back the headers
# in a List<String> so we (obviously) recreate that shape
results[header.lower()] = [value]
resp = json.dumps(results).encode('utf-8')
c_len = len(resp)
self.send_response(200)
self.send_header('content-length', str(c_len))
self.send_header('content-type', 'application/json;charset=utf-8')
self.end_headers()
self.wfile.write(resp)
self.wfile.flush()
except Exception as e:
self.log_error('Bogus: %s', str(e))
self.send_error(500)
self.send_header('content-length', '0')
if __name__ == '__main__':
server_address = ('0.0.0.0', 80)
s = ThreadingHTTPServer(server_address, IntrospectionHandler)
# noinspection PyBroadException
try:
s.serve_forever()
except Exception:
s.shutdown()
class MyNoSQL:
version = "0.0.1"
debuglvl = 7
timeout=600
# dbopen = False
def __init__(self):
# if not self.dbopen:
global db
self.db = {}
self.selfurl = None
self.doc_id = None
self.port = 0
self.dbopen = False
# self.dblocation = os.path.dirname(os.path.realpath(__file__))
self.dblocation = tempfile.gettempdir()
db = self
def debugmsg(self, lvl, *msg):
msglst = []
prefix = ""
suffix = ""
# print("self.debuglvl:",self.debuglvl," >= lvl",lvl," ", self.debuglvl >= lvl)
if self.debuglvl >= lvl:
try:
# print("self.debuglvl:",self.debuglvl," >= 4"," ", self.debuglvl >= 4)
if self.debuglvl >= 4:
stack = inspect.stack()
the_class = stack[1][0].f_locals["self"].__class__.__name__
the_method = stack[1][0].f_code.co_name
the_line = stack[1][0].f_lineno
prefix = "{}: {}({}): [{}:{}] ".format(str(the_class), the_method, the_line, self.debuglvl, lvl)
if len(prefix.strip())<32:
prefix = "{} ".format(prefix)
if len(prefix.strip())<24:
prefix = "{} ".format(prefix)
msglst.append(str(prefix))
suffix = " [{} @{}]".format(self.version, str(datetime.datetime.now().isoformat(sep=' ', timespec='seconds')))
# print("msg:",msg)
for itm in msg:
msglst.append(str(itm))
msglst.append(str(suffix))
print(" ".join(msglst))
except Exception as e:
print("e:",e)
pass
def setdblocation(self, location):
if os.path.isdir(location):
self.dblocation = location
return True
else:
raise Exception("location:", location, "does not exist or is not a folder")
return False
def _checkdb(self):
if "dbpath" not in self.db:
raise Exception("db not opened")
return False
return True
def _servedb(self):
# try to open a port in range 8800 - 8899
for i in range(99):
portno = 8800+i
self.debugmsg(0, "trying port:", portno)
server_address = ('', portno)
reason = ""
if self.dbopen:
try:
self.httpserver = ThreadingHTTPServer(server_address, MyNoSQLServer)
# self.httpserver.setdb(self)
self.port = portno
except PermissionError:
reason = "PermissionError"
except Exception as e:
reason = "{}".format(e)
else:
reason = "DB Closed"
if self.port>0:
break
else:
self.debugmsg(0, "open port failed:", reason)
if self.port>0:
self.doc_id = self._registerself()
self.debugmsg(0, "Server started on port:", self.port)
self.httpserver.serve_forever()
else:
self.debugmsg(0, "Unable to start server:", reason)
def _server(self):
self.dbserver = threading.Thread(target=self._servedb)
self.dbserver.start()
peers = threading.Thread(target=self._findpeers)
peers.start()
while self.dbopen:
reg = threading.Thread(target=self._registerself)
reg.start()
time.sleep(60)
upd = threading.Thread(target=self._peerupdates)
upd.start()
self.httpserver.shutdown()
self.dbserver.join()
def _registerpeer(self, doc_id):
if "peers" not in self.db:
self.db["peers"] = []
if doc_id not in self.db["peers"]:
self.db["peers"].append(doc_id)
def _findpeers(self):
time.sleep(5)
dbservers = self.indexread("dbserver")
self.debugmsg(5, "dbservers:", dbservers)
for dbserver in dbservers:
self.debugmsg(5, "dbserver:", dbserver)
self.debugmsg(5, "dbservers[dbserver]:", dbservers[dbserver])
if dbserver != self.doc_id:
# indexes = self._getremote(dbservers[dbserver] + "/Index")
# self.debugmsg(5, "indexes:", indexes)
dbserverdoc = self.readdoc(dbserver)
if dbserverdoc["dbmode"] != "Peer":
self._registerpeer(dbserver)
def _peerupdates(self):
if "peers" not in self.db:
self.db["peers"] = []
selfdoc = self.getselfdoc()
if selfdoc["dbmode"] == "Peer":
mirrorid = random.choice(self.db["peers"])
self._getpeerupdates(mirrorid, selfdoc["dbmode"])
if selfdoc["dbmode"] == "Mirror":
for peer in self.db["peers"]:
self._getpeerupdates(peer, selfdoc["dbmode"])
def _getpeerupdates(self, doc_id, mode):
self.debugmsg(8, "doc_id:", doc_id)
peerdoc = self.readdoc(doc_id)
if "dbserver" in peerdoc:
peerindexes = self._getremote(peerdoc["dbserver"] + "/Index")
if peerindexes is not None:
self.debugmsg(8, "peerindexes:", peerindexes)
for index in peerindexes:
self._updatepeerindex(peerdoc["dbserver"], index)
if mode == "Mirror":
peerrevs = self._getremote(peerdoc["dbserver"] + "/Index/rev")
self.debugmsg(8, "peerrevs:", peerrevs)
if peerrevs is not None:
t = datetime.datetime.now()
for peerrev in peerrevs:
rdet = self._revdetail(peerrevs[peerrev])
self.debugmsg(7, "rdet:", rdet)
getremote = False
if rdet["epoch"] > (t.timestamp() - ONE_YEAR):
islocal = self._islocal(peerrev)
if islocal is None:
getremote = True
else:
ldet = self._revdetail(islocal)
if rdet["number"] > ldet["number"]:
getremote = True
self.debugmsg(7, "getremote:", getremote)
if getremote:
rdoc = self._getremote(peerdoc["dbserver"] + "/Doc/" + peerrev)
self.debugmsg(7, "rdoc:", rdoc)
self._saveremotedoc(rdoc)
def _updatepeerindex(self, peerurl, index):
self.debugmsg(8, "index:", index)
indexremote = self._getremote(peerurl + "/Index/" + index)
self.debugmsg(8, "index:", index, "indexremote:", indexremote)
indexlocal = self.indexread(index)
self.debugmsg(8, "index:", index, "indexlocal:", indexlocal)
for item in indexremote:
self.debugmsg(8, "item:", item)
if item not in indexlocal.keys():
self._indexadd(index, item, indexremote[item])
if index == "rev":
# compare revisions
rdet = self._revdetail(indexremote[item])
ldet = self._revdetail(indexlocal[item])
if rdet["number"] > ldet["number"]:
self._indexadd(index, indexremote, indexremote[item])
def _getremote(self, uri):
try:
r = requests.get(uri, timeout=self.timeout)
self.debugmsg(9, "resp: ", r.status_code, "r.text:", r.text)
if (r.status_code != requests.codes.ok):
self.debugmsg(9, "r.status_code:", r.status_code, "!=", requests.codes.ok)
return None
else:
if "{" in r.text or "[" in r.text:
jsonresp = json.loads(r.text)
self.debugmsg(9, "jsonresp: ", jsonresp)
return jsonresp
else:
return r.text
except Exception as e:
self.debugmsg(8, "Exception:", e)
return None
def _sendremote(self, uri, payload):
try:
r = requests.post(uri, json=payload, timeout=self.timeout)
self.debugmsg(9, "resp: ", r.status_code, r.text)
if (r.status_code != requests.codes.ok):
self.debugmsg(9, "r.status_code:", r.status_code, "!=", requests.codes.ok)
return None
else:
if "{" in r.text or "[" in r.text:
jsonresp = json.loads(r.text)
self.debugmsg(9, "jsonresp: ", jsonresp)
return jsonresp
else:
return r.text
except Exception as e:
self.debugmsg(8, "Exception:", e)
return None
def addpeer(self, peerurl):
uri = peerurl + "/Peer"
# payload = {
# "AgentName": self.agentname,
# "Action": "Status",
# "Hash": hash
# }
payload = self.getselfdoc()
self.debugmsg(9, "payload: ", payload)
resp = self._sendremote(uri, payload)
self.debugmsg(5, "resp: ", resp)
peerdoc = self._getremote(uri)
self.debugmsg(5, "peerdoc: ", peerdoc)
if "id" in peerdoc:
self._saveremotedoc(peerdoc)
pass
def getselfdoc(self):
if self.dbopen:
if self.doc_id is None:
self.doc_id = self._registerself()
if self.doc_id is not None:
doc = self.readdoc(self.doc_id)
return doc
else:
raise Exception("self.doc_id is None")
return None
else:
raise Exception("DB not open.")
return None
def _registerself(self):
if self.port>0:
doc = None
if self.doc_id is None:
srvdisphost = socket.gethostname()
self.selfurl = "http://{}:{}".format(srvdisphost, self.port)
dbservers = self.indexread("dbserver")
self.debugmsg(7, "dbservers:", dbservers)
if self.selfurl in list(dbservers.values()):
# find doc id
for dbserver in dbservers:
self.debugmsg(5, "dbserver:", dbserver)
self.debugmsg(5, "dbservers[dbserver]:", dbservers[dbserver])
if dbservers[dbserver] == self.selfurl:
self.doc_id = dbserver
self.debugmsg(7, "self.doc_id:", self.doc_id)
doc = self.readdoc(self.doc_id)
else:
# create new server doc
doc = {}
doc["dbserver"] = self.selfurl
doc["altconn"] = []
if "dbmode" not in doc:
doc["dbmode"] = "Peer"
if doc is not None:
t = datetime.datetime.now()
doc["lastregistered"] = t.timestamp()
self.savedoc(doc)
return self.doc_id
def getdbmode(self):
if self.dbopen:
doc = self.getselfdoc()
if "dbmode" in doc:
return doc["dbmode"]
else:
return None
else:
return None
def setdbmode(self, newmode):
# check for supported modes
self.debugmsg(5, "newmode:", newmode)
if newmode not in ["Peer", "Mirror"]:
return False
if self.dbopen:
doc = self.getselfdoc()
if "dbmode" not in doc:
doc["dbmode"] = newmode
self.savedoc(doc)
else:
if doc["dbmode"] is not newmode:
doc["dbmode"] = newmode
self.savedoc(doc)
self.debugmsg(7, "doc:", doc)
return True
else:
return False
def opendb(self, dbname):
self.db["dbpath"] = os.path.join(self.dblocation, dbname)
if not os.path.isdir(self.db["dbpath"]):
os.mkdir(self.db["dbpath"])
self.debugmsg(0, "DB Created:", self.db["dbpath"])
self.index = os.path.join(self.db["dbpath"], "index")
if os.path.isfile(self.index):
self.db["index"] = self._loadindex()
self.debugmsg(0, "DB Opened:", self.db["dbpath"])
else:
self.db["index"] = {}
self.db["index"]["rev"] = {}
self._saveindex()
self.dbopen = True
self.server = threading.Thread(target=self._server)
self.server.start()
# wait till serving
while self.port<8800:
time.sleep(1)
def closedb(self):
self.dbopen = False
self.server.join()
self.db = {}
def _lockaquire(self, filename):
timeout = 60
t = datetime.datetime.now()
timestart = t.timestamp()
lockfile = "{}.lock".format(filename)
while os.path.isfile(lockfile):
timenow = t.timestamp()
if (timenow - timestart)>timeout:
return False
break
self.debugmsg(6, "waiting for lock on", filename)
time.sleep(0.1)
with open(lockfile, 'w') as f:
f.write("{}".format(threading.get_native_id()))
self.debugmsg(9, "lock aquired on", filename)
return True
def _lockrelease(self, filename):
lockfile = "{}.lock".format(filename)
if os.path.isfile(lockfile):
os.remove(lockfile)
self.debugmsg(9, "lock released on", filename)
def _saveindex(self):
if self._lockaquire(self.index):
file = open(self.index, "wb")
c_index = self._compressdata(self.db["index"])
file.write(c_index)
file.close()
self._lockrelease(self.index)
def _loadindex(self):
if self._lockaquire(self.index):
file = open(self.index, "rb")
rawdata = file.read()
file.close()
self._lockrelease(self.index)
self.debugmsg(9, "rawdata:", rawdata)
data = self._decompressdata(rawdata)
return data
def _template0(self):
pass
def _compressdata(self, data):
data_in = bytearray(json.dumps(data).encode("utf8"))
data_out = lzma.compress(data_in)
return data_out
def _decompressdata(self, data_in):
data_out = lzma.decompress(data_in)
data = json.loads(data_out)
return data
def _generatedocid(self):
return str(uuid.uuid1())
def _updaterev(self, doc):
irev = 0
t = datetime.datetime.now()
if "rev" in doc:
srev = doc["rev"].split(".", 1)[0]
irev = int(srev)
ts = t.timestamp()
tss = int(ts)
tsm = int("{}".format(ts - tss).split(".")[1][0:6])
doc["rev"] = "{}.{:x}.{:x}".format(irev+1, tss, tsm)
self.debugmsg(8, "doc[rev]:", doc["rev"])
return doc
def _revdetail(self, rev):
detail = {}
detail["string"] = rev
arev = rev.split(".")
detail["number"] = int(arev[0])
# hex_val = 'beef101'
# print(int(hex_val, 16))
detail["epoch"] = float("{}.{}".format(int(arev[1], 16), int(arev[2], 16)))
return detail
def _checkrev(self, doc):
if "id" not in doc:
return True
id = doc["id"]
if not self._docindexed(id):
return True
if "rev" not in doc:
return False
if self._islocal(id) is None:
return True
odoc = self.readdoc(id)
if doc["rev"] != odoc["rev"]:
cdet = self._revdetail(doc["rev"])
# self.debugmsg(5, "cdet:", cdet)
odet = self._revdetail(odoc["rev"])
# self.debugmsg(5, "odet:", odet)
# self.debugmsg(5, "type(odet[epoch]):", type(odet["epoch"])) type(odet[epoch]): <class 'float'>
if cdet["number"] > odet["number"]:
return True
raise Exception("Document revisions don't match:", doc["rev"], "!=", odoc["rev"])
return False
return True
def _indexadd(self, key, doc_id, value):
changed = False
# self.debugmsg(9, "key:", key)
if key not in self._indexlist():
self.indexadd(key)
# self.debugmsg(9, "doc_id:", doc_id)
if doc_id not in self.db["index"][key].keys():
self.db["index"][key][doc_id] = value
changed = True
# self.debugmsg(9, "value:", value)
if self.db["index"][key][doc_id] != value:
self.db["index"][key][doc_id] = value
changed = True
# self.debugmsg(9, "changed:", changed)
if changed:
self._saveindex()
def _indexdoc(self, doc):
changed = False
kidx = list(self.db["index"].keys())
kidx.remove("local")
for key in kidx:
if key in doc:
if doc["id"] not in self.db["index"][key] or self.db["index"][key][doc["id"]] != doc[key]:
self.db["index"][key][doc["id"]] = doc[key]
changed = True
if changed:
self._saveindex()
def _indexlist(self):
idxlst = list(self.db["index"].keys())
# self.debugmsg(9, "idxlst:", idxlst)
return idxlst
def indexadd(self, index):
if index not in self.db["index"]:
self.db["index"][index] = {}
self._saveindex()
return True
def indexdrop(self, index):
if index in self.db["index"]:
del self.db["index"][index]
self._saveindex()
return True
def indexread(self, index):
if index in self.db["index"]:
return self.db["index"][index]
else:
self.indexadd(index)
self.indexupdate()
return self.db["index"][index]
def indexupdate(self):
docs = self.db["index"]["rev"].keys()
for id in docs:
self.readdoc(id)
def _docindexed(self, doc_id):
rev = self.indexread("rev")
if doc_id in list(rev.keys()):
return True
return False
def _dochash(self, doc):
hasher = hashlib.md5()
# hasher.update(str(os.path.getmtime(file)).encode('utf-8'))
hasher.update(json.dumps(doc).encode("utf8"))
return hasher.hexdigest()
def _saveremotedoc(self, doc):
t = datetime.datetime.now()
self._checkrev(doc)
if "id" not in doc:
doc["id"] = self._generatedocid()
doc_id = doc["id"]
if "documents" not in self.db:
self.db["documents"] = {}
if doc["id"] not in self.db["documents"]:
self.db["documents"][doc["id"]] = {}
if "dochash" not in self.db["documents"][doc["id"]] or \
self.db["documents"][doc["id"]]["dochash"] != self._dochash(doc):
self.db["documents"][doc["id"]]["data"] = doc
self.db["documents"][doc["id"]]["accessed"] = t.timestamp()
self.db["documents"][doc["id"]]["dochash"] = self._dochash(doc)
self._savetoshard(doc["id"])
self._indexdoc(doc)
return doc
def savedoc(self, doc):
t = datetime.datetime.now()
self._checkrev(doc)
if "id" not in doc:
doc["id"] = self._generatedocid()
doc_id = doc["id"]
if "documents" not in self.db:
self.db["documents"] = {}
if doc["id"] not in self.db["documents"]:
self.db["documents"][doc["id"]] = {}
if "dochash" not in self.db["documents"][doc["id"]] or \
self.db["documents"][doc["id"]]["dochash"] != self._dochash(doc):
doc = self._updaterev(doc)
self.db["documents"][doc["id"]]["data"] = doc
self.db["documents"][doc["id"]]["accessed"] = t.timestamp()
self.db["documents"][doc["id"]]["dochash"] = self._dochash(doc)
self._savetoshard(doc["id"])
self._indexdoc(doc)
return doc
def readdoc(self, doc_id):
doc = None
islocal = self._islocal(doc_id)
self.debugmsg(8, "doc_id:", doc_id, "islocal:", islocal)
if islocal is not None:
t = datetime.datetime.now()
if "documents" not in self.db:
self.db["documents"] = {}
if doc_id not in self.db["documents"]:
# need to load the doc from the shard
shard_id = self._getshardid(doc_id)
if "shards" not in self.db:
self.db["shards"] = {}
if shard_id not in self.db["shards"]:
self.db["shards"][shard_id] = self._loadshard(shard_id)
self.db["documents"][doc_id] = {}
self.db["documents"][doc_id]["data"] = self.db["shards"][shard_id][doc_id]
self.db["documents"][doc_id]["accessed"] = t.timestamp()
self.db["documents"][doc_id]["dochash"] = self._dochash(doc)
self._freeshard(shard_id)
doc = self.db["documents"][doc_id]["data"]
self.db["documents"][doc_id]["accessed"] = t.timestamp()
self._indexdoc(doc)
return doc
def _getshardid(self, doc_id):
# shard_id = doc_id.split("-")[0]
# I have been comtemplating how big to make the shard_id
# it's a compromise between having too many files or too big files.
# Initially I was going to make the the first 2 char of the uuid
# but then I made it the first part (8 char) which might make too many files?
# Further thought 16 x 16 (2 char) = 256 files, this was the windows limit at
# one stage. might still be?
# Found the answer, FAT16 limit is 512 files in a folder and a 3 char shard
# 16 x 16 x 16 = 4096 would well exceed this, but would be fine on FAT32 or NTFS
# https://stackoverflow.com/questions/4944709/windows-limit-on-the-number-of-files-in-a-particular-folder#14407078
shard_id = doc_id[0:2]
return shard_id
def _islocal(self, doc_id):
if "local" not in self.db["index"]:
self.db["index"]["local"] = {}
# self.debugmsg(9, "index local keys:", self.db["index"]["local"].keys())
# if doc_id in self.db["index"]["local"].keys():
# self.debugmsg(6, "doc_id:", doc_id, ":", self.db["index"]["local"][doc_id])
# return self.db["index"]["local"][doc_id]
if doc_id in self.db["index"]["local"].keys():
del self.db["index"]["local"][doc_id]
shard_id = self._getshardid(doc_id)
# self.debugmsg(9, "shard_id:", shard_id, "doc_id:", doc_id)
if "shards" not in self.db:
self.db["shards"] = {}
if shard_id not in self.db["shards"]:
self.db["shards"][shard_id] = self._loadshard(shard_id)
# self.debugmsg(9, "shard_id:", shard_id, ":", self.db["shards"][shard_id])
if len(self.db["shards"][shard_id])>0:
for id in list(self.db["shards"][shard_id].keys()):
# self.debugmsg(9, "id:", id, ":", self.db["shards"][shard_id][id])
self.db["index"]["local"][id] = self.db["shards"][shard_id][id]["rev"]
self._freeshard(shard_id)
else:
self._freeshard(shard_id)
return None
if doc_id in self.db["index"]["local"].keys():
return self.db["index"]["local"][doc_id]
return None
def _savetoshard(self, doc_id):
if "local" not in self.db["index"]:
self.db["index"]["local"] = {}
shard_id = self._getshardid(doc_id)
self.debugmsg(7, "shard_id:", shard_id, " doc_id:", doc_id)
if "shards" not in self.db:
self.db["shards"] = {}
if shard_id not in self.db["shards"]:
self.db["shards"][shard_id] = self._loadshard(shard_id)
self.db["shards"][shard_id][doc_id] = self.db["documents"][doc_id]["data"]
self.db["index"]["local"][doc_id] = self.db["shards"][shard_id][doc_id]["rev"]
self._saveshard(shard_id)
def _saveshard(self, shard_id):
c_shard = self._compressdata(self.db["shards"][shard_id])
f_shard = os.path.join(self.db["dbpath"], shard_id)
if self._lockaquire(f_shard):
file = open(f_shard, "wb")
file.write(c_shard)
file.close()
self._lockrelease(f_shard)
del self.db["shards"][shard_id]
def _loadshard(self, shard_id):
f_shard = os.path.join(self.db["dbpath"], shard_id)
data = {}
if os.path.isfile(f_shard):
if self._lockaquire(f_shard):
file = open(f_shard, "rb")
rawdata = file.read()
file.close()
self._lockrelease(f_shard)
data = self._decompressdata(rawdata)
return data
def _freeshard(self, shard_id):
del self.db["shards"][shard_id]
def DisplayInterface(params):
# Store the menu in a variable so as to provide easy access at any point in time.
menu = f"""
* To clear all structure files: {c.OKGREEN}-R{c.ENDC}
* To revert post timestamps: {c.OKGREEN}-r{c.ENDC}
* To display this menu: {c.WARNING}-h{c.ENDC}
* To host local web server to preview local site: {c.WARNING}-p{c.ENDC}
* To host public web server to preview local site: {c.WARNING}-P{c.ENDC}
* To exit this mode and build the site: {c.FAIL}exit{c.ENDC}
* To exit this mode and quit the program: {c.FAIL}!exit{c.ENDC}
"""
# If the terminal window is less than 59 characters wide, resize the menu
# to better fit.
rows, columns = popen('stty size', 'r').read().split()
if (int(columns) < 59):
menu = sub(":\s+", ":\n ", menu)
# Continue prompting the user for input until they enter a valid argument
while (True):
if ("-a" in params): # Entered CLI. Prompt for command.
print(menu)
params = GetUserInput("#: ")
if ("-R" in params): # Rebuild all structure files
print(" - Clearing structure files... ", end="", flush=True)
for file in listdir("./html"):
if (file.endswith(".html")):
remove("./html/"+file)
for file in listdir("./html/blog"):
if (file.endswith(".html")):
remove("./html/blog/"+file)
print(f"{c.OKGREEN}done.{c.ENDC}")
break
elif ("-r" in params): # Revert post timestamps
print(" - Reverting post timestamps... ", end="", flush=True)
for files in listdir("./content"):
if (files.endswith(".txt")):
Revert("./content/"+files)
print(f"{c.OKGREEN}done.{c.ENDC}")
elif ("-h" in params or "help" in params): # Print help menu
print(f'Entering "-h" at any time will display the menu below.\n{menu}')
elif ("-p" in params or "-P" in params): # Web server
from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
# Clear log file
open("./server.log", "w").close()
# Setup the request handler
class GetHandler(BaseHTTPRequestHandler):
def do_GET(self): # Handle GET requests
# Transform request for root to request for index.html
if (self.path == "/"):
self.path = "/index.html"
# Strip leading / from request
resource = self.path[1:]
# Test for file existence
if (isfile(f"./html/{resource}")):
# If file exists, send 200 code and appropriate content
# header based on file type.
self.send_response(200)
extension = resource.rsplit(".", 1)[-1]
if (extension == "css"):
self.send_header('Content-Type','text/css; charset=utf-8')
elif (extension == "html"):
self.send_header('Content-Type','text/html; charset=utf-8')
elif (extension == "xml"):
self.send_header('Content-Type','text/xml; charset=utf-8')
elif (extension == "jpg"):
self.send_header('Content-Type','image/jpg')
elif (extension == "ico"):
self.send_header('Content-Type','image/ico')
else:
self.send_header('Content-Type','text/plain; charset=utf-8')
self.end_headers()
# Serve file
fd = open(f"./html/{resource}", "rb")
for i,line in enumerate(fd):
self.wfile.write(line)
fd.close()
# If file does not exist, send 404 code and serve 404 page.
else:
self.send_response(404)
self.send_header('Content-Type','text/html; charset=utf-8')
self.end_headers()
fd = open(f"./html/404.html", "rb")
for i,line in enumerate(fd):
self.wfile.write(line)
fd.close()
# Send nothing if client uses a valid but unsupported HTTP methods.
def do_HEAD(self): return False
def do_POST(self): return False
def do_PUT(self): return False
def do_DELETE(self): return False
def do_CONNECT(self): return False
def do_OPTIONS(self): return False
def do_TRACE(self): return False
def do_PATCH(self): return False
# Custom log handler that prints log message and writes to log file.
def log_request(self, code):
server_fd = open("./server.log", "a")
server_fd.write(f"{self.client_address[0]} - - [{self.log_date_time_string()}] {self.requestline} {code} -\n")
server_fd.close()
print(f"{self.client_address[0]} - - [{self.log_date_time_string()}] {self.requestline} {code} -")
# Make web server public or private, and notify user
if ("-P" in argv):
server_address = ("0.0.0.0", 8000)
print(f"Serving {c.WARNING}public{c.ENDC} web server at port {c.OKGREEN}8000{c.ENDC}. Use {c.BOLD}CTRL-C{c.ENDC} to exit.")
else:
server_address = ("127.0.0.1", 8000)
print(f"Serving {c.OKGREEN}private{c.ENDC} web server at port {c.OKGREEN}8000{c.ENDC}. Use {c.BOLD}CTRL-C{c.ENDC} to exit.")
# Create the server, and serve it until the user issues an interrupt.
httpd = ThreadingHTTPServer(server_address, GetHandler)
try:
open_new_tab("http://localhost:8000")
httpd.serve_forever()
# On interrupt, close the log file and gracefully shutdown web server.
except KeyboardInterrupt:
print(f"\r{c.OKGREEN}Exiting.{c.ENDC}")
server_fd = open("./server.log", "a")
server_fd.write(f"- - - [{localtime().tm_mday}/{localtime().tm_mon}/{localtime().tm_year} {localtime().tm_hour}:{localtime().tm_min}:{localtime().tm_sec}] Graceful shutdown - -\n")
server_fd.close()
httpd.shutdown()
exit(0)
elif ("!exit" in params): # Exit without building site
exit(0)
else: # Exit then build site
break
# Unless "--exit" flag is supplied, prompt user for more input.
if ("--exit" in params):
exit(0)
params = GetUserInput("#: ")
def do_GET(self):
self._handle('do_get')
def do_POST(self):
self._handle('do_post')
# Start Server
print('Starting server on', port_number)
print('Control-C to stop.')
server = ThreadingHTTPServer(('', port_number), RequestHandler)
t = ServerThread(server)
t.start()
try:
while True:
time.sleep(60)
except KeyboardInterrupt:
print('Keyboard Interrupt: shutting down server.')
server.shutdown()
t.join(timeout=shutdown_timeout_s)
if t.is_alive():
print('Server is not shut down after timeout period.')
else:
print('Server shut down.')
WAR)
proto = 'http'
else:
logger.log(
'Usando CERT:("%s") y KEY:("%s") para usar el protocolo HTTPS ...'
% (global_conf.ssl['cert'], global_conf.ssl['key']), PER)
proto = 'https'
logger.log('Escuchando en :: %s://%s:%d/%s' % (proto, LHOST, LPORT, RPATH),
PER)
try:
httpd.serve_forever()
except KeyboardInterrupt:
pass
httpd.shutdown()
logger.log('Saliendo ...', debug.INF)
if (get_os_name == 'posix'):
reset_term.reset()
def startServer():
try:
server = ThreadingHTTPServer(('',80), RequestHandler)
server.serve_forever()
except KeyboardInterrupt:
server.shutdown()
class ImageWindow(QWidget):
def __init__(self, config, parent=None):
super().__init__(parent)
self.config = config
self.setWindowTitle('3D Vorschau')
vbox = QVBoxLayout()
# buttons for global operations, modes
save = QPushButton("Speichern")
save.clicked.connect(self.saveTriggered)
rotleft = QPushButton("Linkes drehen")
rotleft.clicked.connect(self.config.leftState().rotate90)
rotright = QPushButton("Rechtes drehen")
rotright.clicked.connect(self.config.rightState().rotate90)
self._linkedBtn = QCheckBox("verbunden", self)
self._linkedBtn.setChecked(self.config.linked)
self._linkedBtn.stateChanged.connect(self.config.setLinked)
self.config.linkedChanged.connect(self.setLinked)
self._modeGroup = QButtonGroup(self)
self._modeGroup.addButton(QRadioButton("skalieren",self),0)
self._modeGroup.addButton(QRadioButton("schieben",self),1)
self._modeGroup.addButton(QRadioButton("drehen",self),2)
self._modeGroup.addButton(QRadioButton("drehen+skalieren",self),3)
self._modeGroup.button(self.config.mode).setChecked(True)
self._modeGroup.buttonClicked['int'].connect(self.config.setMode)
self.config.modeChanged.connect(self.setMode)
tb = QHBoxLayout()
tb.addWidget(save)
tb.addStretch(1)
tb.addWidget(rotleft)
tb.addWidget(rotright)
tb.addWidget(self._linkedBtn)
tb.addWidget(self._modeGroup.button(0))
tb.addWidget(self._modeGroup.button(1))
tb.addWidget(self._modeGroup.button(2))
tb.addWidget(self._modeGroup.button(3))
vbox.addLayout(tb)
# the views to the left and right image
self.leftImage = ImageView(self.config.leftState(), self.config, False)
self.rightImage = ImageView(self.config.rightState(), self.config, True)
imgbox = QHBoxLayout()
imgbox.setSpacing(0)
imgbox.addStretch(1)
imgbox.addWidget(self.leftImage)
imgbox.addWidget(self.rightImage)
imgbox.addStretch(1)
vbox.addLayout(imgbox)
self.setLayout(vbox)
self.leftImage.mousePress.connect(self.mousePressL)
self.leftImage.mouseMove.connect(self.mouseMove)
self.rightImage.mousePress.connect(self.mousePressR)
self.rightImage.mouseMove.connect(self.mouseMove)
self.httpd = ThreadingHTTPServer(('', 12345), ImageWebserver)
self.httpd.appconfig = self.config
self.httpd_thread = threading.Thread(target=self.httpd.serve_forever)
self.httpd_thread.daemon = True
self.httpd_thread.start()
def closeEvent(self, e):
self.httpd.shutdown()
self.httpd.server_close()
def setLinked(self, value):
self._linkedBtn.setChecked(value!=0)
self._linkedBtn.repaint()
def setMode(self, value):
self.config.setLinked(value==0)
self._modeGroup.button(value).setChecked(True)
def computeScale(self, start, end):
try:
p1 = start - self._mouseCenter
p2 = end - self._mouseCenter
s = math.sqrt(float(p2.x()*p2.x()+p2.y()*p2.y())/
float(p1.x()*p1.x()+p1.y()*p1.y()))
return QTransform().scale(s,s)
except ZeroDivisionError:
return QTransform()
def computeMove(self, start, end):
try:
m = QPointF(end-start)/self.leftImage.imgRect().width()
return QTransform(1,0,0,1,m.x(),m.y())
except ZeroDivisionError:
return QTransform()
def computeRotate(self, start, end):
try:
p1 = start - self._mouseCenter
p2 = end - self._mouseCenter
return QTransform().rotateRadians(math.atan2(p1.x(),p1.y()) - math.atan2(p2.x(),p2.y()))
except ZeroDivisionError:
return QTransform()
def computeRotateScale(self, start, end):
try:
p1 = start - self._mouseCenter
p2 = end - self._mouseCenter
s = float(p1.y()*p2.x()-p1.x()*p2.y())/float(p1.x()*p1.x()+p1.y()*p1.y())
c = (p2.y()+p1.x()*s)/p1.y()
return QTransform(c,-s,s,c,0,0)
except ZeroDivisionError:
return QTransform()
def mousePressL(self, m):
self._mouseCenter = self.leftImage.imgRect().center()
self.mousePress(m,0)
def mousePressR(self, m):
self._mouseCenter = self.rightImage.imgRect().center()
self.mousePress(m,1)
def mousePress(self, m, side):
self._mouseStart = m.pos()
self._mouseSide = side
self._leftTransform = self.config.leftState().transform
self._rightTransform = self.config.rightState().transform
def mouseMove(self, m):
self._mouseEnd = m.pos()
if self.config.mode == 0:
transform = self.computeScale(self._mouseStart, self._mouseEnd)
elif self.config.mode == 1:
transform = self.computeMove(self._mouseStart, self._mouseEnd)
elif self.config.mode == 2:
transform = self.computeRotate(self._mouseStart, self._mouseEnd)
elif self.config.mode == 3:
transform = self.computeRotateScale(self._mouseStart, self._mouseEnd)
if (self._mouseSide==0 or self.config.linked):
self.config.leftState().transform = self._leftTransform * transform
if (self._mouseSide==1 or self.config.linked):
self.config.rightState().transform = self._rightTransform * transform
def saveTriggered(self):
dstFile = self.config.proposeFilename()
if dstFile == None: return
print("saving in "+dstFile)
# create an image for the final output
img = QImage(self.config.saveSize, QImage.Format_RGB888)
qp = QPainter(img)
# black background and then the images
dst = QRect(0,0,img.width(), img.height())
brush = QBrush(Qt.SolidPattern)
brush.setColor(Qt.black)
qp.setBrush(brush)
qp.drawRect(dst)
self.config.paintImage(qp, dst)
qp.end()
img.save(dstFile)
class BrowserIntegration(QtCore.QObject):
listen_port_changed = QtCore.pyqtSignal(int)
def __init__(self, parent=None):
super().__init__(parent)
self.server = None
@property
def host_address(self):
if not self.server:
return ''
return self.server.server_address[0]
@property
def port(self):
if not self.server:
return 0
return self.server.server_address[1]
@property
def is_running(self):
return self.server is not None
def start(self):
if self.server:
self.stop()
config = get_config()
if config.setting["browser_integration_localhost_only"]:
host_address = '127.0.0.1'
else:
host_address = '0.0.0.0' # nosec
try:
for port in range(config.setting["browser_integration_port"],
65535):
try:
self.server = ThreadingHTTPServer((host_address, port),
RequestHandler)
except OSError:
continue
log.info("Starting the browser integration (%s:%d)",
host_address, port)
self.listen_port_changed.emit(port)
threading.Thread(target=self.server.serve_forever).start()
break
else:
log.error(
"Failed finding an available port for the browser integration."
)
self.stop()
except Exception:
log.error("Failed starting the browser integration on %s",
host_address,
exc_info=True)
def stop(self):
if self.server:
try:
log.info("Stopping the browser integration")
self.server.shutdown()
self.server.server_close()
self.server = None
self.listen_port_changed.emit(self.port)
except Exception:
log.error("Failed stopping the browser integration",
exc_info=True)
else:
log.debug("Browser integration inactive, no need to stop")
class Gui(threading.Thread):
def __init__(self, width, height):
'''
Creating the GUI
:return:
:rtype:
'''
threading.Thread.__init__(self)
self.connected = False
self.led_array_listener = []
self.connection_state_listener = []
self.width = width
self.height = height
# Register listener in the LocalCGIHTTPRequestHandler class
LocalCGIHTTPRequestHandler.local_cgi_script[
"connection_status"] = self.add_connection_state_listener
LocalCGIHTTPRequestHandler.local_cgi_script[
"led_array"] = self.add_led_array_listener
# Create server instance
self.https = ThreadingHTTPServer(('', 8000),
LocalCGIHTTPRequestHandler)
self.led_array = []
self.html_led_array = 'EMPTY'
def run(self):
self.set_connected(True)
self.https.serve_forever()
def end(self):
self.set_connected(False)
self.https.shutdown()
self.https.server_close()
def set_array_colors(self, led_array):
self.led_array = deepcopy(led_array)
html = ['<div id="ledArray2">']
for row in self.led_array:
html.append('<div class="line">')
for r, g, b in row:
html.append(
f'<div class="led" style="background-color:rgb({r}, {g}, {b});"></div>'
)
html.append('</div>')
html.append('</div>')
self.html_led_array = ''.join(html)
self.inform_led_array_listener()
def add_connection_state_listener(self, q):
self.connection_state_listener.append(q)
self.inform_connection_state_listener(q)
def add_led_array_listener(self, q):
self.led_array_listener.append(q)
self.inform_led_array_listener(q)
@staticmethod
def get_color_in_hex(color):
c = '#'
for pure in color:
c += f'{pure:02X}'[-2:]
return c
def inform_connection_state_listener(self, q=None):
if q:
lst = [q]
else:
lst = self.connection_state_listener
for l in lst:
l.put("Connected" if self.connected else "NOT Connected")
if not self.connected:
l.put("#QUIT#")
def inform_led_array_listener(self, q=None):
if q:
lst = [q]
else:
lst = self.led_array_listener
for l in lst:
if not self.connected:
l.put("#QUIT#")
else:
l.put(self.html_led_array)
def set_connected(self, connected):
self.connected = connected
self.inform_connection_state_listener()
self.inform_led_array_listener()
