def set_volume(self, val):
if val < 0:
val = 0
elif val > 1:
val = 1
self._mod_volume = val
logger.d('set volume = {}'.format(val))
def load_configuration():
""" load configuration. """
global CONF
CONF = read_configuration_file(DEF_CONF_FILE)
logger.d("app.py:load_configuration:CONF: {0}".format(CONF))
# Validation Configuration values
if CONF.saved_conf_filename is None:
CONF.saved_conf_filename = './changed_logstash.conf'
if CONF.log_file is None or not len(CONF.log_file):
CONF.log_file = './rl_watcher.log'
if CONF.period is None:
CONF.period = 60
if CONF.logstash is None or not len(CONF.logstash):
return 1
else:
if CONF.logstash.sentinel is None or not len(CONF.logstash.sentiel):
return 2
if CONF.logstash.ssh is None or not len(CONF.logstash.sentinel):
CONF.logstash.ssh.host = "127.0.0.1"
CONF.logstash.ssh.port = 22
if CONF.conf_path is None or len(CONF.conf_path):
CONF.conf_path = '/etc/logstash/conf.d/logstash.conf'
logger.d("complete %s configuration" % DEF_CONF_FILE)
def run(self):
logo.dodo()
ur = update_rate
while not self.stopped:
if 1 > datetime.now().hour > 6:
if ur == update_rate:
ur = 0
json_data = vk.Wall.get()
likes = vk.Wall.get_latest_post_likes(json_data)
fromid = int(vk.Wall.get_from_id(json_data))
post_id = vk.Wall.get_latest_post(json_data)
if not vk.Likes.is_liked(post_id):
if likes < min_likes:
if fromid_group == fromid:
delay = rand(delay_from, delay_to)
log.d("From dodo")
else:
delay = rand(big_delay_from, big_delay_to)
log.d("Not from dodo")
log.i("Delay = " + str(delay))
wait(delay)
else:
log.i("Delay = 0")
log.i_n("Likes = " + str(vk.Likes.add(post_id)) +
"\n***Text***\n" +
vk.Wall.get_latest_post_text(json_data) +
"\n**********")
ur += 1
wait(1)
def __init__(self, pool_num=0):
self.pool_num_ = pool_num if pool_num > 0 else cpu_count()
self.q = Queue.Queue()
self.__work = None
self.__complated = None
self.abort = False
logger.d('set pool = {}'.format(self.pool_num_))
def __restart():
""" restart logstash.
@return: result string
"""
global ssh_host, ssh_port
location = __get_location()
cmd_list = []
if location == DEF_REMOTE:
tmp = __get_ssh_command()
cmd_list.append(tmp)
tmp = 'service logstash restart'
cmd_list.append(tmp)
cmd = ' '.join(cmd_list)
logger.d('CMD: %s' % cmd)
try:
if sys.version_info < (3,):
o = sp.check_output(cmd, shell=True)
else:
o = sp.getoutput(cmd)
except:
logger.w(sys.exc_info()[0])
return o
def __restart():
""" restart logstash.
@return: result string
"""
global ssh_host, ssh_port
location = __get_location()
cmd_list = []
if location == DEF_REMOTE:
tmp = __get_ssh_command()
cmd_list.append(tmp)
tmp = 'service logstash restart'
cmd_list.append(tmp)
cmd = ' '.join(cmd_list)
logger.d('CMD: %s' % cmd)
try:
if sys.version_info < (3, ):
o = sp.check_output(cmd, shell=True)
else:
o = sp.getoutput(cmd)
except:
logger.w(sys.exc_info()[0])
return o
def run(self):
p = Pool(self.pool_num_)
p.apply(self.__worker)
logger.d('closing sound pool.')
p.close()
p.terminate()
p.join()
logger.d('closed sound pool.')
def do_work(self, arg=None):
if self.__work is None:
return
try:
if arg is None:
res = self.__work()
else:
res = self.__work(arg)
self.do_complated(res)
except Exception:
logger.e('Unexpected error: {}'.format(str(sys.exc_info()[0])))
logger.e(traceback.format_exc())
except KeyboardInterrupt:
logger.d('keyboard Ctrl+C in sound_pool.do_work()')
self.abort = True
raise
def read_configuration_file(file_name):
""" open configuration file.
@param file_name: (str) configuration file name
@return (json object) environment
"""
logger.d("[read_configuration_file prcoess]")
logger.d("filename -> %s" % file_name)
try:
fp = codecs.open(file_name, mode='rb', encoding='utf-8')
c = fp.read()
fp.close()
except:
print("Error: ", sys.exc_info()[0])
jo = jsontree.loads(c, encoding='utf-8')
return jo
def __ctrl_sound(self):
is_brake = False
# stop action
if self._event_stop.is_set():
logger.d('stop sound.')
is_brake = True
# pause action
if self._event_pause.is_set():
logger.d('pause sound.')
while self._event_pause.is_set():
time.sleep(0.1)
if self._event_stop.is_set():
# finish
break
is_brake = False
return is_brake
def cli(debug, service, conf_path):
""" command line interface. """
DEF_CONF_FILE = conf_path
fp = codecs.open(DEF_CONF_FILE, 'rb', encoding='utf-8')
conf_contents = fp.read()
fp.close()
cf = jsontree.loads(conf_contents, encoding="utf-8")
logger.d("log filename: {0}".format(cf.log_file))
if cf.log_file is not None:
log_file = open(cf.log_file, 'w')
if service:
daemonize(debug, conf_path, log_file)
else:
main(debug, False)
def __worker(self):
while True:
# logger.d('queue size = {}'.format(self.q.qsize()))
try:
if self.abort:
return
req = self.q.get(timeout=3)
self.do_work(req)
logger.d('kick work = {}'.format(req))
self.q.task_done()
except Queue.Empty:
time.sleep(0.05)
except Queue.Full:
continue
except KeyboardInterrupt:
self.abort = True
raise
except Exception:
raise
def send(self, data: bytes) -> None:
'''
Send the bytes over the connection
'''
i = 0
one_percent = ((len(data) / self._MSS) // 100) or 1
while i < len(data):
with self._receive_condition:
# Wait until the window is open
while i + self._MSS - self._last_ack > self._MWS:
self._receive_condition.wait()
segment_data = data[i:i + self._MSS]
self._send_segment(data=segment_data)
logger.add_file_size(len(segment_data))
i += self._MSS
if (i / self._MSS) % one_percent == 0:
s = 'Progress: {:0.2f}%, seq_num: {}, estimated_rtt: ' + \
'{:0.2f}, dev_rtt: {:0.2f}'
s = s.format(i * 100 / len(data), self._seq_num,
self._estimatedRTT, self._devRTT)
logger.d(s)
def __set_configuration(filename):
""" set configuration of logstash at path. """
global conf_path, src_path
location = __get_location()
if location == DEF_REMOTE:
cmd = __get_scp_command()
if location == DEF_LOCAL:
cmd = 'cp -f {0} {1}'.format(src_path, conf_path)
logger.d('CMD: %s' % cmd)
try:
if sys.version_info < (3, ):
o = sp.check_output(cmd, shell=True)
else:
sp.getoutput(cmd)
except:
logger.w(sys.exc_info()[0])
def process(debug_mode):
""" process. """
global conf, src_path
logger.DEBUG_MODE = debug_mode
conf = __get_configuration()
logger.d("configuration: " + json.dumps(conf))
redis_info = __get_host_info()
logger.d("redis info: {0}".format(str(redis_info)))
for i in redis_info:
chg_conf = __change_config(i, master)
if chg_conf is not None:
f = codecs.open(src_path, 'wb', encoding='utf-8')
f.write(chg_conf)
f.close()
__set_configuration(src_path)
o = __restart()
logger.i(o)
def __set_configuration(filename):
""" set configuration of logstash at path. """
global conf_path, src_path
location = __get_location()
if location == DEF_REMOTE:
cmd = __get_scp_command()
if location == DEF_LOCAL:
cmd = 'cp -f {0} {1}'.format(src_path, conf_path)
logger.d('CMD: %s' % cmd)
try:
if sys.version_info < (3,):
o = sp.check_output(cmd, shell=True)
else:
sp.getoutput(cmd)
except:
logger.w(sys.exc_info()[0])
def run(self):
logger.i('I started')
self.__st.cht_ready(self)
while True:
# Main loop for receiving requests
try:
req = self.__scrp_sock.receive_request()
# Dispatch
rht = RequestHandlerThread(self, req)
logger.d('Starting RequestHandlerThread {}'.format(rht.name))
rht.start()
except BytesMessageReceiveError as e:
# Connection is broken
break
except (MessageDecodeError,
JsonParseError,
InvalidRequestDictError) as e:
# Bad request
raise BadRequestError
except ScrpError as e:
logger.e(e)
logger.d('Thread terminate')
def chagne_logstash_conf(master_ip, master_port):
""" chagne logstash configuration.
@param master_ip: (str) redis master ip.
@param master_port: (str) redis master port.
"""
global CONF, DEF_DEBUG_MODE
logger.d("[change_logstash_conf prcoess]")
cls = CONF.logstash
# ### Init Logstash
ssh = cls.ssh
host = ssh['host']
conf_path = cls.conf_path
filename = CONF.saved_conf_filename
logstash.src_path = filename
logstash.master = (master_ip, master_port)
if cls.ssh.id_file is not None or cls.ssh.id_file != "":
ssh_id_filename = cls.ssh.id_file
if ssh_id_filename == '':
logstash.ssh_id_file = None
else:
logstash.ssh_id_file = ssh_id_filename
if ssh.user is not None:
ssh_user = ssh.user
if ssh_user == '':
logstash.ssh_user = None
else:
logstash.ssh_user = ssh_user
logstash.ssh_host = host
logstash.conf_path = conf_path
logstash.process(DEF_DEBUG_MODE)
def process(debug_mode):
""" process. """
global conf, src_path
logger.DEBUG_MODE = debug_mode
conf = __get_configuration()
logger.d("configuration: " + json.dumps(conf))
redis_info = __get_host_info()
logger.d("redis info: {0}".format(str(redis_info)))
for i in redis_info:
chg_conf = __change_config(i, master)
if chg_conf is not None:
f = codecs.open(src_path, 'wb', encoding='utf-8')
f.write(chg_conf)
f.close()
__set_configuration(src_path)
o = __restart()
logger.i(o)
def __get_host_info():
""" get host information.
@return: (list) [(host, port), ...] for redis
"""
global conf
hp = r'input.+?[{].+?redis.+?[{].+?host.+?=>.+?"(.+?)".+?\n'
hosts = re.findall(hp, conf, re.S)
logger.d("current redis hosts: " + str(hosts))
pp = r'input.+?[{].+?redis.+?[{].+?port.+?=>.+?(\d{1,5}).+?\n'
ports = re.findall(pp, conf, re.S)
logger.d("current redis ports: " + str(ports))
if len(hosts) != len(ports):
logger.w('different count hosts and ports')
return []
ips = []
for i in range(len(hosts)):
ips.append((hosts[i], int(ports[i])))
return ips
def __get_host_info():
""" get host information.
@return: (list) [(host, port), ...] for redis
"""
global conf
hp = r'input.+?[{].+?redis.+?[{].+?host.+?=>.+?"(.+?)".+?\n'
hosts = re.findall(hp, conf, re.S)
logger.d("current redis hosts: " + str(hosts))
pp = r'input.+?[{].+?redis.+?[{].+?port.+?=>.+?(\d{1,5}).+?\n'
ports = re.findall(pp, conf, re.S)
logger.d("current redis ports: " + str(ports))
if len(hosts) != len(ports):
logger.w('different count hosts and ports')
return []
ips = []
for i in range(len(hosts)):
ips.append((hosts[i], int(ports[i])))
return ips
def __playsound(self, wavfile, loop=False):
if (wavfile == ""):
logger.d('empty sound file.')
return
# open file
logger.d('open file = {}'.format(wavfile))
wf = wave.open(wavfile, 'rb')
self.wfinfo = wf.getparams()
self.show_wavinfo(self.wfinfo)
try:
with SoundPlayer.__pa_lock:
p_ = pyaudio.PyAudio()
s_ = p_.open(format=p_.get_format_from_width(self.wfinfo[1]),
channels=self.wfinfo[0],
rate=self.wfinfo[2],
output=True)
# play stream
input_data = wf.readframes(CHUNK)
logger.d('started blocking sound play.')
while len(input_data) > 0:
if self.__ctrl_sound():
break
s_.write(self.__mod_sound(input_data))
input_data = wf.readframes(CHUNK)
# loop
if loop and len(input_data) == 0:
wf.rewind()
input_data = wf.readframes(CHUNK)
finally:
# close stream
with SoundPlayer.__pa_lock:
s_.stop_stream()
s_.close()
wf.close()
p_.terminate()
logger.d('finished sound play.')
def get_response(url, log_tag): # returns txt json
k = 1
while k <= 3:
try:
log.d(log_tag + " try " + str(k) + "/3")
response = requests.get(url).text
log.d(log_tag + " complete")
k = 111
return response
except Exception:
k += 1
log.d(log_tag + " error")
try:
f = open("stub.json")
stub = f.read()
f.close()
return stub
except Exception:
for i in range(3):
log.d_n("FUUUUUUUUCK!!!!!!!!")
def __get_configuration():
""" get configuration of logstash.
@return: (str) configuration contents
"""
global ssh_host, ssh_port, ssh_id_file, conf_path
location = __get_location()
logger.d("logstash:__get_configuration:location: {0}".format(location))
cmd_list = []
if location == DEF_REMOTE:
tmp = __get_ssh_command()
cmd_list.append(tmp)
tmp = 'cat {0}'.format(conf_path)
cmd_list.append(tmp)
cmd = " ".join(cmd_list)
logger.d('CMD: %s' % cmd)
o = None
if cmd != '':
try:
logger.d("logstash:__get_configuration:try")
if sys.version_info < (3,):
o = sp.check_output(cmd_list, shell=True)
else:
o = sp.getoutput(cmd)
except:
logger.w("logstash:__get_configuration:except")
logger.w(sys.exc_info()[0])
return o
def __get_configuration():
""" get configuration of logstash.
@return: (str) configuration contents
"""
global ssh_host, ssh_port, ssh_id_file, conf_path
location = __get_location()
logger.d("logstash:__get_configuration:location: {0}".format(location))
cmd_list = []
if location == DEF_REMOTE:
tmp = __get_ssh_command()
cmd_list.append(tmp)
tmp = 'cat {0}'.format(conf_path)
cmd_list.append(tmp)
cmd = " ".join(cmd_list)
logger.d('CMD: %s' % cmd)
o = None
if cmd != '':
try:
logger.d("logstash:__get_configuration:try")
if sys.version_info < (3, ):
o = sp.check_output(cmd_list, shell=True)
else:
o = sp.getoutput(cmd)
except:
logger.w("logstash:__get_configuration:except")
logger.w(sys.exc_info()[0])
return o
def _init_pool(cls):
SoundInterface.pool.set_work(SoundInterface.__post_push)
SoundInterface.pool.set_complated(
lambda x: logger.d('post result = {}'.format(x)))
def send(seg: Segment, addr, events=None) -> None:
if not seg.data:
# Bypass PLD for non-data segments
_actually_send(seg, addr, events=events)
return
logger.d('processing segment {}'.format(seg))
global _reordered
events = events or []
logger.add_pld_seg()
# Check if it's time to send reordered
if _reordered:
_reordered[0] -= 1
if not _reordered[0]:
_actually_send(_reordered[1], addr, events=['rord'])
_reordered = None
# Get the segment as bytes
seg_bytes = bytearray(seg.to_bytes())
if random.random() < _args.pDrop:
logger.d('DROPPED {}'.format(seg.seq_num))
log(['drop'], seg)
logger.add_transmitted() # Include dropped packets as transmitted
return
if random.random() < _args.pDuplicate:
logger.d('DUPLICATED {}'.format(seg.seq_num))
_actually_send(seg, addr, events=events)
logger.add_pld_seg()
events.append('dup')
elif random.random() < _args.pCorrupt:
logger.d('CORRUPTED {}'.format(seg.seq_num))
# Choose a random bit in the data
flip = int(random.random() * 8 * len(seg.data))
# Extract the bit in the byte
bit = 1 << (flip % 8)
# 12 bytes header followed by the data segment
seg_bytes[12 + flip // 8] ^= bit # flip the bit
# Add the corruption event
events.append('corr')
elif random.random() < _args.pOrder and not _reordered:
logger.d('REORDERED {}'.format(seg.seq_num))
_reordered = [_args.maxOrder, seg]
return
elif random.random() < _args.pDelay:
logger.d('DELAYED {}'.format(seg.seq_num))
events.append('dely')
t = Timer(random.random() * _args.maxDelay / 1000,
lambda: _actually_send(seg, addr, events=events))
t.daemon = True
t.start()
return
_actually_send(seg, addr, seg_bytes=seg_bytes, events=events)
def put(self, item, block=True, timeout=None):
self.q.put(item, block, timeout)
logger.d('put {}'.format(item))
def show_wavinfo(self, wfinfo):
logger.d(wfinfo)
logger.d('wave file info')
logger.d('channels = {}'.format(wfinfo[0]))
logger.d('sampling width = {} byte'.format(wfinfo[1]))
logger.d('frame rate = {} Hz'.format(wfinfo[2]))
logger.d('frame count = {}'.format(wfinfo[3]))
logger.d('sound time = {} s'.format((int)(wfinfo[3] / wfinfo[2])))
def cht_ready(self, cht: 'ClientHandlerThread'):
"""Inform that a cht is ready"""
with self.__lock:
logger.d(str(cht))
self.__cht_set.add(cht)
def main(debug_mode, repeat):
""" main process. """
global CONF, DEF_DEBUG_MODE
DEF_DEBUG_MODE = debug_mode
logger.DEBUG_MODE = debug_mode
load_configuration()
logstash = CONF.logstash
sentinel_num = len(logstash.sentinel)
if logstash.sentinel_master is not None:
DEF_MASTER_ID = logstash.sentinel_master
while True:
logger.d("")
logger.d("=" * 80)
logger.d("[Loop Watch] Start")
logger.d("-" * 80)
i = logstash.sentinel[random.randint(0, sentinel_num - 1)]
logger.d("sentinel info: {0}".format(i))
s = sentinel.get_session(i.host, i.port)
logger.d("sentinel session: {0}".format(s))
m = sentinel.get_master(s, DEF_MASTER_ID)
logger.d("master info: {0}".format(m))
# ### Change logstash configuration
if m is not None:
chagne_logstash_conf(m[0], m[1])
else:
logger.w("Check Master Name or Slave Failed")
if repeat:
time.sleep(float(CONF.period))
else:
break
logger.d("-" * 80)
logger.d("[Loop Watch] End")