def check_for_interrupted_scan(config):
logger = logging.getLogger('changeme')
if config.fresh:
logger.debug("Forcing a fresh scan")
remove_queues()
elif config.resume:
logger.debug("Resuming previous scan")
return
if os.path.isfile(PERSISTENT_QUEUE):
scanners = FIFOSQLiteQueue(path=".",
multithreading=True,
name="scanners")
fingerprints = FIFOSQLiteQueue(path=".",
multithreading=True,
name="fingerprints")
logger.debug("scanners: %i, fp: %i" %
(scanners.qsize(), fingerprints.qsize()))
if scanners.qsize() > 0 or fingerprints.qsize() > 0:
if not prompt_for_resume(config):
remove_queues()
else:
remove_queues()
fp = RedisQueue('fingerprint')
scanners = RedisQueue('scanners')
fp_qsize = 0
scanners_qsize = 0
logger.debug('scanners: %s, fp: %s' % (scanners_qsize, fp_qsize))
try:
fp_qsize = fp.qsize()
except redis.exceptions.ConnectionError:
pass
try:
scanners_qsize = scanners.qsize()
except redis.exceptions.ConnectionError:
pass
if fp_qsize > 0 or scanners_qsize > 0:
if not prompt_for_resume(config):
remove_queues()
if fp_qsize == 0 and scanners_qsize == 0:
# Clear the found queue if there's no fingerprints or scanners in the queues
try:
logger.debug('Clearing found_q')
found_q = RedisQueue('found_q')
found_q.delete()
except Exception as e:
logger.debug('Exception: %s: %s' %
(type(e).__name__, e.__str__().replace('\n', '|')))
pass
def _get_queue(self, name):
try:
# Try for redis
r = RedisQueue(name)
r.ping()
self.logger.debug('Using RedisQueue for %s' % name)
return r
except redis.ConnectionError:
# Fall back to sqlite persistent queue
self.logger.debug('Using FIFOSQLiteQueue for %s' % name)
return FIFOSQLiteQueue(path='.', multithreading=True, name=name)
def run(host, port, debug):
global apps
global queue
global worker
apps = store.get("apps", {})
queue = FIFOSQLiteQueue(path=config.queue_db, multithreading=True)
worker = Worker(queue, store)
if config.provider_use_queue:
worker.start()
server.run(host=host, port=port, debug=debug)
from flask import request
from flask_api import FlaskAPI, status, exceptions
from datetime import datetime
from persistqueue import FIFOSQLiteQueue, exceptions
q = FIFOSQLiteQueue('my_queue', multithreading=True, auto_commit=True)
app = FlaskAPI(__name__)
"""
(Persist-Queue)[https://github.com/peter-wangxu/persist-queue/blob/master/persistqueue/exceptions.py]
(Flask API)[https://www.flaskapi.org]
- [x] Create endpoints for POST and GET
- [ ] Think about DELETE...
"""
@app.route('/', methods=['GET', 'POST'])
def manage_queue():
"""
Pop or Put message in the Queue
"""
if request.method == 'POST':
# Create a new queue entry
data = {
'message': request.data.get('message', ''),
'created_at': datetime.now()
}
q.put(data)
return f'OK {len(q)} in QUEUE', status.HTTP_201_CREATED
else:
# request.method == 'GET'
try:
from persistqueue import FIFOSQLiteQueue
from threading import Thread
q = FIFOSQLiteQueue(path="c:\\uconnect_logs", multithreading=True)
def worker():
while True:
print('QSize before get', q.size)
#q.tran_lock.acquire()
item = q.get()
#q.tran_lock.release()
print(item, 'current qsize', q.size)
if __name__ == "__main__":
for item in range(30):
q.put({'Item #': item})
for i in range(3):
t = Thread(target=worker)
t.daemon = True
t.start()
t.join()
for item in source():
q.put(item)
q.join()
#Example usage for SQLite3 based queue
def worker():
while True:
item = q.get()
print(item)
q = FIFOSQLiteQueue(path='c:\uconnect_logs', multithreading=True)
items = [{'Item': 1}, {'Item': 2}, {'Item': 3}, {'Item': 4}, {'Item': 5}]
num_worker_threads = 3
for i in range(num_worker_threads):
t = Thread(target=worker)
t.daemon = True
t.start()
for item in items:
q.put(item)
q.join()
from persistqueue import FIFOSQLiteQueue
from threading import Thread
def goc_queue(name='default'):
qname = os.path.expanduser(os.path.join(_ROOT, f'{name}-{_QUEUE}'))
os.makedirs(os.path.dirname(qname), mode=0o775, exist_ok=True)
return FIFOSQLiteQueue(path=qname)
class request_object:
def __init__(self, command, path):
self.command = command
self.path = path
class retry_object:
def __init__(self, request):
self.request = request
self.remaining_tries = 7
self.last_run_time = time.time()
# FIFO queues
requestsQ = FIFOSQLiteQueue(path="./requestsQ", multithreading=True)
retryQ = FIFOSQLiteQueue(path="./retryQ", multithreading=True)
# Send a get request to a server address
def sendget(request):
try:
address = request.path.split("/")[1]
path = request.path.replace("/" + address + "/", "/")
print("http://" + address + path)
r = requests.get("http://" + address + path)
return r.status_code
except:
return '503'