def __init__(self, redis_conn, workflow_id, app_id, config, lang='en'):
"""Initialize the minion."""
Minion.__init__(self, redis_conn, workflow_id, app_id, config)
self.terminate_proc_queue = multiprocessing.Queue()
self.execute_process = None
self.ping_process = None
self.module = None
self._state = {}
self.config = config
self.transpiler = ScikitLearnTranspiler(config)
configuration.set_config(self.config)
self.tmp_dir = self.config.get('config', {}).get('tmp_dir', '/tmp')
sys.path.append(self.tmp_dir)
signal.signal(signal.SIGTERM, self._terminate)
self.mgr = socketio.RedisManager(
config['juicer']['servers']['redis_url'], 'job_output')
self.executor = ThreadPoolExecutor(max_workers=1)
self.job_future = None
self.scikit_learn_config = config['juicer'].get('scikit_learn', {})
# self termination timeout
self.active_messages = 0
self.self_terminate = True
self.juicer_listener_enabled = False
self.current_lang = lang
def _update_notification_count(config):
mgr = socketio.RedisManager(config['servers']['redis_url'][:-1],
'job_output')
mgr.emit('notifications',
data={'unread': _get_number_of_unread_notifications()},
room='user:' + flask_globals.user.id,
namespace=NAMESPACE)
def do_simulated_tasks(sid):
external_sio = socketio.RedisManager('redis://mq:6379/0', write_only=True)
for i in range(10):
external_sio.emit('results', data={'random_number': i}, room=sid)
time.sleep(1)
return 'Complete'
def __init__(self, redis_conn, workflow_id, app_id, config, lang='en'):
Minion.__init__(self, redis_conn, workflow_id, app_id, config)
self.terminate_proc_queue = multiprocessing.Queue()
self.execute_process = None
self.ping_process = None
self.reload_code_process = None
self.module = None
self._state = {}
self.transpiler = SparkTranspiler(config)
self.config = config
configuration.set_config(self.config)
self.juicer_listener_enabled = False
self.tmp_dir = self.config.get('config', {}).get('tmp_dir', '/tmp')
sys.path.append(self.tmp_dir)
self.mgr = socketio.RedisManager(
config['juicer']['servers']['redis_url'], 'job_output')
self.executor = ThreadPoolExecutor(max_workers=1)
self.job_future = None
# self termination timeout
self.active_messages = 0
self.self_terminate = True
# Errors and messages
self.MNN000 = ('MNN000', _('Success.'))
self.MNN001 = ('MNN001', _('Port output format not supported.'))
self.MNN002 = ('MNN002', _('Success getting data from task.'))
self.MNN003 = ('MNN003', _('State does not exists, processing app.'))
self.MNN004 = ('MNN004', _('Invalid port.'))
self.MNN005 = ('MNN005',
_('Unable to retrieve data because a previous error.'))
self.MNN006 = ('MNN006',
_('Invalid Python code or incorrect encoding: {}'))
self.MNN007 = ('MNN007', _('Job {} was canceled'))
self.MNN008 = ('MNN008', _('App {} was terminated'))
self.MNN009 = ('MNN009', _('Workflow specification is missing'))
# Used in the template file, declared here to gettext detect them
self.msgs = [
_('Task running'),
_('Task completed'),
_('Task ignored (not used by other task or as an output)')
]
self.current_lang = lang
# self._build_dist_file()
signal.signal(signal.SIGTERM, self._terminate)
signal.signal(signal.SIGINT, self._cleanup)
self.last_job_id = 0
self.new_session = False
self.cluster_options = {}
self.last_cluster_id = None
self.DIST_ZIP_FILE = '/tmp/lemonade-lib-pythoni_{}.zip'.format(
self.app_id)
def xsend_message():
import random
mq = socketio.RedisManager()
mq.emit("my_response", {'data': "HEEEEE_LLOOOO {}".format(random.randint(100, 999))})
mq.emit("notifications", {'data': "notifications {}".format(random.randint(100, 999))})
sio = socketio.Client()
sio.connect('http://localhost:8080')
sio.emit("notifications", {'data': "XXXX notifications {}".format(random.randint(100, 999))})
def wrap_socketio_server():
import socketio
from websocket_server.ws_conn import WSConnections
mgr = socketio.RedisManager("redis://localhost:%s/0" % redis_port)
sio = socketio.Server(client_manager=mgr, async_mode='eventlet')
#init
ws = WSConnections.getInstance(sio)
ws.init_events()
app = socketio.Middleware(sio, _app)
return app
def create_socket_io_app(_app):
"""
Creates websocket app
:param _app: Flask app
"""
original_emit = socketio.base_manager.BaseManager.emit
socketio.base_manager.BaseManager.emit = mocked_emit(original_emit, _app)
socket_io_config = _app.config['STAND_CONFIG']['servers']
mgr = socketio.RedisManager(socket_io_config['redis_url'], 'job_output')
sio = socketio.Server(engineio_options={'logger': True},
client_manager=mgr,
cors_allowed_origins='*',
allow_upgrades=True)
return sio, socketio.Middleware(sio, _app)
def lib_send_redis_message(message, save_flag=True):
try:
# connect to the redis queue
redis = socketio.RedisManager(config['REDIS_REMOTE_URL'],
channel=config['SOCKET_IO_CHANNEL'],
write_only=True)
# emit an event
redis.emit('msg',
data=message,
namespace=config['SOCKET_IO_NAMESPACE'])
except:
logger.error("ERROR! Cannot connect to {}".format(
config['REDIS_REMOTE_URL']))
return False
if (save_flag):
lib_save_message_history(message)
def __init__(self):
mgr = client_manager = socketio.RedisManager("redis://")
Payload.max_decode_packets = 500
self.socketio = socketio.Server(async_mode="gevent",
client_manager=mgr)
self.app = socketio.WSGIApp(self.socketio)
self.r = redis.Redis(host='localhost', port=6379, db=0)
self.connectionlist = {
} #json.loads(self.r.get("connectionlist").decode())
#fill redis
self.r.set("connectionlist", json.dumps(self.connectionlist))
self.r.set(
"image",
"https://cdna.artstation.com/p/assets/images/images/023/154/550/large/nikolai-lockertsen-f0952e7c-f76f-4899-94f0-a475241af0a3.jpg"
)
self.tag = "networking"
self.sidinfo = {}
# classes
self.motd = motd()
# stuff
self.tasklist = [] # currently working on
def post():
result, result_code = dict(
status="ERROR",
message=gettext('Missing json in the request body')), 400
if request.json is not None:
socket_io_config = current_app.config['STAND_CONFIG']['servers']
mgr = socketio.RedisManager(socket_io_config['redis_url'],
'job_output')
sio = socketio.Server(engineio_options={'logger': True},
client_manager=mgr,
cors_allowed_origins='*',
allow_upgrades=True)
sio.emit(event=request.json.get("event", "?"),
data=request.json.get("data", {}),
room=request.json.get("room", "general"),
namespace=request.json.get("namespace", "/stand"))
result_code = 200
result = {"status": "OK"}
return result, result_code
def __init__(self,
config,
minion_executable,
log_dir='/tmp',
config_file_path=None):
self.minion_support_process = None
self.new_minion_watch_process = None
self.start_process = None
self.minion_status_process = None
self.state_control = None
self.minion_watch_process = None
self.active_minions = {}
self.config = config
configuration.set_config(config)
self.config_file_path = config_file_path
self.minion_executable = minion_executable
self.log_dir = log_dir or self.config['juicer'].get('log', {}).get(
'path', '/tmp')
signal.signal(signal.SIGTERM, self._terminate)
self.port_range = list(
range(*(config['juicer'].get('minion', {}).get(
'libprocess_port_range', [36000, 36500]))))
self.advertise_ip = config['juicer'].get(
'minion', {}).get('libprocess_advertise_ip')
# Minion requires 3 different ports:
# 1 for libprocess/Mesos communication
# 1 for driver port
# 1 for block manager
self.port_offset = config['juicer'].get('minion',
{}).get('port_offset', 100)
self.mgr = socketio.RedisManager(
config['juicer']['servers']['redis_url'], 'job_output')
import socketio
import json
from .log import logger
from .dbHelper import r_userTokens, configData
from .utils import getUID
from .user import signout
mgr = socketio.RedisManager("redis://" + configData["redis"]["host"] + ":" +
str(configData["redis"]["port"]) + "/" +
str(configData["redis"]["websocketsDB"]))
sio = socketio.Server(
async_mode="gevent_uwsgi",
cors_allowed_origins="*",
logger=True,
engineio_logger=True,
client_manager=mgr,
)
@sio.event
def connect(sid, environ):
logger.info("New WS socket connected: " + sid)
@sio.event
def disconnect(sid):
logger.info("WS socket disconnected: " + sid)
@sio.event
def __init__(self):
mgr = socketio.RedisManager('redis://')
self.socketio = socketio.Server(client_manager=mgr)
self.r = redis.Redis(host="localhost", port=6379, db=0)
self.tag = "events"
def simulate():
logging.config.fileConfig('logging_config.ini')
logger = logging.getLogger(__name__)
# ap = argparse.ArgumentParser()
# ap.add_argument('-c', '')
mgr = socketio.RedisManager(app.config.get('REDIS_URL'), 'job_output')
statuses = [
StatusExecution.RUNNING,
# StatusExecution.CANCELED, StatusExecution.ERROR,
# StatusExecution.PENDING, StatusExecution.INTERRUPTED,
StatusExecution.WAITING,
StatusExecution.COMPLETED
]
while True:
try:
_, job_json = redis_store.blpop('queue_start')
job = json.loads(job_json)
logger.debug('Simulating workflow %s with job %s',
job.get('workflow_id'), job.get('job_id'))
eventlet.sleep(3)
for k in ['job_id']:
if k in job:
logger.info('Room for %s', k)
room = str(job[k])
mgr.emit('update job',
data={
'message': random.choice(MESSAGES),
'status': StatusExecution.RUNNING,
'id': job['workflow_id']
},
room=room,
namespace="/stand")
job_entity = Job.query.get(job.get('job_id'))
job_entity.status = StatusExecution.RUNNING
job_entity.finished = datetime.datetime.utcnow()
db.session.add(job_entity)
db.session.commit()
for task in job.get('workflow', {}).get('tasks', []):
if task['operation']['id'] == 25: # comment
continue
job_step_entity = JobStep.query.filter(
and_(JobStep.job_id == job.get('job_id'),
JobStep.task_id == task['id'])).first()
# Updates task in database
try:
job_step_entity.status = StatusExecution.RUNNING
job_step_entity.logs.append(
JobStepLog(level='WARNING',
date=datetime.datetime.now(),
message=random.choice(MESSAGES)))
db.session.add(job_step_entity)
db.session.commit()
except Exception as ex:
logger.error(ex)
for k in ['job_id']:
if k in job:
logger.info('Room for %s and task %s', k,
task.get('id'))
room = str(job[k])
mgr.emit('update task',
data={
'message': random.choice(MESSAGES),
'status': random.choice(statuses[:-2]),
'id': task.get('id')
},
room=room,
namespace="/stand")
eventlet.sleep(random.randint(2, 5))
for k in ['job_id']:
if k in job:
room = str(job[k])
mgr.emit('update task',
data={
'message': random.choice(MESSAGES),
'status': StatusExecution.COMPLETED,
'id': task.get('id')
},
room=room,
namespace="/stand")
# Updates task in database
try:
# Visualizations
if task['operation']['id'] in [35, 68, 69, 70, 71]:
# import pdb
# pdb.set_trace()
for k in ['job_id']:
room = str(job[k])
mgr.emit('task result',
data={
'msg': 'Result generated',
'status': StatusExecution.COMPLETED,
'id': task['id'],
'task': {
'id': task['id']
},
'title': 'Table with results',
'type': 'VISUALIZATION',
'operation': {
'id': task['operation']['id']
},
'operation_id':
task['operation']['id']
},
room=room,
namespace="/stand")
#
# result = JobResult(task_id=task['id'],
# title="Table with results",
# operation_id=task['operation']['id'],
# type=ResultType.VISUALIZATION, )
# logger.info('Result created for job %s', job['job_id'])
# job_entity.results.append(result)
job_step_entity.status = StatusExecution.COMPLETED
job_step_entity.logs.append(
JobStepLog(level='WARNING',
date=datetime.datetime.now(),
message=random.choice(MESSAGES)))
db.session.add(job_step_entity)
except Exception as ex:
logger.error(ex)
# eventlet.sleep(5)
for k in ['job_id']:
if k in job:
logger.info('Room for %s', k)
room = str(job[k])
mgr.emit('update job',
data={
'message': random.choice(MESSAGES),
'status': StatusExecution.COMPLETED,
'finished': job_entity.finished.isoformat(),
'id': job['job_id']
},
room=room,
namespace="/stand")
if job_entity:
job_entity.status = StatusExecution.COMPLETED
db.session.add(job_entity)
db.session.commit()
except KeyError as ke:
logger.error('Invalid json? KeyError: %s', ke)
raise
except Exception as ex:
logger.error(ex.message)
raise
logger.info('Simulation finished')
from flask_socketio import SocketIO
from .models.const import creatives
from .models.user import User
from .libs.utils import now_date
from pyhottop.pyhottop import Hottop
# from libs.hottop_thread import Hottop
import copy
import eventlet
import logging
import os
import random
import socketio
import sys
from redis import Redis
mgr = socketio.RedisManager('redis://' + os.environ.get('REDIS_HOST'))
sio = SocketIO(client_manager=mgr)
login_manager = LoginManager()
mongo = PyMongo()
ht = Hottop()
logger = logging.getLogger("cloud_cafe")
logger.setLevel(logging.DEBUG)
shandler = logging.StreamHandler(sys.stdout)
fmt = '\033[1;32m%(levelname)-5s %(module)s:%(funcName)s():'
fmt += '%(lineno)d %(asctime)s\033[0m| %(message)s'
shandler.setFormatter(logging.Formatter(fmt))
logger.addHandler(shandler)
eventlet.monkey_patch()
import socketio
import eventlet
eventlet.monkey_patch()
mgr = socketio.RedisManager('redis://socketio-redis:6379/0')
#sio = socketio.Server(cors_allowed_origins='*',async_mode='gevent',client_manager=mgr)
sio = socketio.Server(logger=True,
engineio_logger=True,
cors_allowed_origins='*',
async_mode='eventlet',
client_manager=mgr)
app = socketio.WSGIApp(sio)
@sio.event
def connect(sid, environ):
print('connect ', sid)
@sio.event
def test_message(sid, data):
print('message ', data)
sio.emit('message', {'data': 'pong'}, broadcast=True, include_self=True)
mgr.emit('ping message', data={'foo': 'bar'}, room=sid)
# sio.emit('my event', {'data': 'foobar'}, room=user_sid)
@sio.event
def ping_message(sid, data):
print('message ping ', data)
import traceback
import socketio
import json
import logging
from celery import Celery
from celery.bin import worker
from . import REDIS_URL, UNBABEL_API_KEY, UNBABEL_USERNAME, init_logging
from .unbabel_api import UnbabelApi, UnauthorizedException, BadRequestException
from . import db
LOGGER = logging.getLogger('translator.tasks')
app = Celery('translator', broker=REDIS_URL)
sio = socketio.Server(client_manager=socketio.RedisManager(REDIS_URL),
write_only=True)
unbabel = UnbabelApi(username=UNBABEL_USERNAME,
api_key=UNBABEL_API_KEY,
sandbox=True)
def event(channel, sid, message):
LOGGER.warning(message)
sio.emit(channel, message, room=sid)
def status_event(sid, translation):
event('status', sid, json.dumps(translation.__dict__))
def error_event(sid, error):
import redis
# r = redis.Redis(
# host='127.0.0.1',
# port='6379',
# password='')
# r.set('ad', 'bar')
# value = r.keys("*")
# print(value)
import socketio
from flask import Flask, render_template
mgr = socketio.RedisManager('redis://')
sio = socketio.Server(client_manager=mgr, async_mode='threading', logger=True, engineio_logger=True)
app = Flask(__name__)
app.wsgi_app = socketio.WSGIApp(sio, app.wsgi_app)
# @sio.event
# def salutations(sid, data):
# print(f"my event is {sid}, {data}")
def update_driver(driver_id):
""" Updates one driver """
print(f"driver-{driver_id} changed!")
@sio.on('position')
def position(sid,driver_id, data):
# TODO: this crashes in debug
import eventlet
eventlet.monkey_patch()
from flask import Flask, render_template
from flask_socketio import SocketIO, join_room, emit, send, leave_room
import logging
import os
import pickle
import socketio as sio
app = Flask(__name__)
app.config['SECRET_KEY'] = 'TODO:ChangeThisSecret' # TODO: change
redis_url = 'redis://*****:*****@app.route('/')
def index():
"""Serve the index HTML"""
return render_template('index.html')
@socketio.on('connect')
def test_connect():
print('Client connected')
emit('my response', {'data': 'Connected'})
def do_revoke(sid, uuid):
cel.control.revoke(uuid)
external_sio = socketio.RedisManager('redis://mq:6379/0', write_only=True)
external_sio.emit('celery task cancelled', data={'uuid': uuid}, room=sid)
return 'Cancelled'
import socketio
import rx
#mgr = socketio.RedisManager('redis://socketio-redis:6379')
#mgr = socketio.RedisManager('redis://socketio-redis:6379/0',channel='chat')
mgr = socketio.RedisManager('redis://socketio-redis:6379/0', write_only=True)
import time
def send(val):
print('sssss')
#sio.emit('message', {'data': val}, broadcast=True, include_self=True)
#mgr.emit('message', data={'foo': 'bar'})
#print(val)
#source = rx.interval(1).subscribe(send)
from datetime import datetime
while True:
time.sleep(2)
now = datetime.now()
current_time = now.strftime("%H:%M:%S")
mgr.emit('message', data={'time': current_time})
from django.core.management.base import BaseCommand
import socketio
import eventlet
eventlet.monkey_patch()
mgr = socketio.RedisManager('redis://localhost:6379/0')
sio = socketio.Server(cors_allowed_origins='*',async_mode='eventlet',client_manager=mgr)
app = socketio.WSGIApp(sio)
@sio.event
def connect(sid, environ):
print('connect ', sid)
@sio.event
def disconnect(sid):
print('disconnect ', sid)
class Command(BaseCommand):
def handle(self, *args, **options):
print('Starting socket server')
eventlet.wsgi.server(eventlet.listen(('', 5001)), app)
from flask import Flask, render_template
import sys
import batch
sys.path.append('./config')
from pyredis import Redis
from db import DB
from config import load_config
try:
db = DB()
redis = Redis()
redis_config = load_config()['redis']
if redis_config['password']:
manager = socketio.RedisManager('redis://:' +
redis_config['password'] + '@' +
redis_config['host'] + ':' +
str(redis_config['port']))
else:
manager = socketio.RedisManager('redis://' + redis_config['host'] +
':' + str(redis_config['port']))
queue = []
sio = socketio.Server(client_manager=manager)
app = Flask('socketio')
except Exception as e:
raise e
@sio.on('connect', namespace='/enliple')
def connect(sid, environ):
logging.debug('connect')
from bitcoin.messages import msg_block, MsgSerializable
from bitcoin.core import b2lx, lx, uint256_from_str, CBlock
from bitcoin.core.serialize import uint256_from_str, uint256_to_str, uint256_from_compact
from bitcoin.wallet import CBitcoinAddress
from bitcoin.core.script import CScript
from bitcointx.wallet import CBitcoinAddress as TX_CBitcoinAddress
from datetime import datetime, timedelta
if not os.path.exists('/data/explorer/blocks/'):
os.makedirs('/data/explorer/blocks/')
if not os.path.exists('/data/explorer/chainstate/'):
os.makedirs('/data/explorer/chainstate')
# connect to the redis queue as an external process
external_sio = socketio.RedisManager('redis://%s' % settings.REDIS_HOST, write_only=True)
def int_to_bytes(i: int, *, signed: bool = False) -> bytes:
length = (i.bit_length() + 7 + int(signed)) // 8
return i.to_bytes(length, byteorder='big', signed=signed)
def bytes_to_int(b: bytes, *, signed: bool = False) -> int:
return int.from_bytes(b, byteorder='big', signed=signed)
def ser_uint256(i):
return uint256_to_str(i)
class TxIdx(object):
def __init__(self, blkhash, spentmask=0):
self.blkhash = blkhash
def __init__(self):
mgr = client_manager = socketio.RedisManager("redis://")
self.socketio = socketio.Server(client_manager=mgr)
self.r = redis.Redis(host='localhost', port=6379, db=0)
self.tag = "modules"
import eventlet
eventlet.monkey_patch()
import logging
import socketio
import json
from . import tasks, db
from . import REDIS_URL, init_logging
LOGGER = logging.getLogger('translator.api')
sio = socketio.Server(async_mode='eventlet', \
client_manager=socketio.RedisManager(REDIS_URL), \
message_queue=REDIS_URL)
app = socketio.WSGIApp(sio,
static_files={
'/': {
'content_type': 'text/html',
'filename': 'static/index.html'
},
'/Index.js': {
'content_type': 'application/javascript',
'filename': 'static/Index.js'
}
})
@sio.on('translate')
def message(sid, data):
LOGGER.info('MESSAGE RECEIVED %s %s ' % (sid, data))
import socketio
from celery import shared_task
from django.conf import settings
mgr = socketio.RedisManager(settings.SOCKET_BROKER_URL, write_only=True)
import json
@shared_task
def update_users_online():
from .models import UserProfile
print('Sending update online command!')
for user in UserProfile.objects.filter(is_online=True):
for room in user.get_sids():
print(room)
mgr.emit('update_online_user',
data={"message": "do update!"},
room=room)
@shared_task
def update_board(board_id):
from .models import Board
from chess.serializers import BoardSerializer
board = Board.objects.get(pk=board_id)
print('Sending update board command!')
try:
for room in board.agressor.get_sids():
print(room)
mgr.emit('update_board',
data=BoardSerializer(board, user=board.agressor).data,
room=room)
def do_mp4(sid, uuid):
external_sio = socketio.RedisManager('redis://mq:6379/0', write_only=True)
return 'Complete'
import os
import socketio
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ahome.settings")
manager = socketio.RedisManager('')
sio = socketio.Server(client_manager=manager)
