def main(unused_argv):
LOGGER.info('load vocab')
vocab = data.Vocab(FLAGS.vocab_path, FLAGS.vocab_max_size)
batch_size = FLAGS.batch_size
if FLAGS.mode == 'decode':
batch_size = FLAGS.beam_size
hyper_params = HYPER_PARAMS(
mode=FLAGS.mode,
batch_size=batch_size,
num_hidden=FLAGS.num_hidden,
emb_dim=FLAGS.emb_dim,
enc_layers=FLAGS.enc_layers,
enc_timesteps=FLAGS.enc_timesteps,
dec_timesteps=FLAGS.dec_timesteps,
max_grad_norm=FLAGS.max_grad_norm,
num_softmax_samples=FLAGS.num_softmax_samples,
min_input_len=FLAGS.min_input_len,
min_lr=FLAGS.min_lr,
lr=FLAGS.lr
)
batch_reader = BatchReader(
data_path=FLAGS.data_path,
vocab=vocab,
hyper_params=hyper_params,
article_key=FLAGS.article_key,
abstract_key=FLAGS.abstract_key,
max_article_sentences=FLAGS.max_article_sentences,
max_abstract_sentences=FLAGS.max_abstract_sentences,
bucketing=FLAGS.use_bucketing,
truncate_input=FLAGS.truncate_input
)
tf.set_random_seed(FLAGS.random_seed)
config = tf.ConfigProto(
gpu_options={"allow_growth": True}, # 按需增长
device_count={"GPU": 2}, # limit to 2 GPU usage
allow_soft_placement=True,
inter_op_parallelism_threads=1, # Nodes that perform blocking operations are enqueued on a pool
intra_op_parallelism_threads=2 # The execution of an individual op (for some op types)
)
if FLAGS.mode == 'train':
model = Seq2SeqAttentionModel(hyper_params, vocab, num_gpus=FLAGS.num_gpus)
_train(model, config, batch_reader)
elif FLAGS.mode == 'eval':
model = Seq2SeqAttentionModel(hyper_params, vocab, num_gpus=FLAGS.num_gpus)
_eval(model, config, batch_reader, vocab)
elif FLAGS.mode == 'decode':
decode_mdl_hps = hyper_params
decode_mdl_hps.dec_timesteps = 1
model = Seq2SeqAttentionModel(decode_mdl_hps, vocab, num_gpus=FLAGS.nu_gpus)
decoder = BeamSearchDecoder(model, batch_reader, hyper_params, vocab)
decoder.decode_loop()
else:
LOGGER.error('not supported mode: %s', hyper_params.mode)
def convert_text_to_binary():
"""convert text data to binary
input data format:
each line looks like:
article=<d> <p> <s> word1 word2 ... </s> <s> ... </s> </p> ... </d>\tabstract=<d> <p> <s> ... </s> </p> ... </d>
"""
text_data_path = FLAGS.in_file
binary_data_path = FLAGS.out_file
assert text_data_path and binary_data_path, 'filename of text data or binary data should be provided'
if not gfile.Exists(binary_data_path):
LOGGER.debug('convert text to binary format: %s => %s', text_data_path, binary_data_path)
reader = open(text_data_path, mode='rb')
writer = open(binary_data_path, mode='wb')
for line in reader:
tf_example = example_pb2.Example()
for feature in line.strip().split(FLAGS.feature_separator):
(k, v) = feature.split('=')
tf_example.features.feature[k].bytes_list.value.extend([v])
tf_example_str = tf_example.SerializeToString()
str_len = len(tf_example_str)
writer.write(struct.pack('q', str_len))
writer.write(struct.pack('%ds' % str_len, tf_example_str))
writer.close()
reader.close()
else:
LOGGER.error('binary data exist: %s', binary_data_path)
def _watch_threads(self):
while True:
time.sleep(60)
input_threads = []
for t in self._input_threads:
if t.is_alive():
input_threads.append(t)
else:
LOGGER.error('found input thread dead')
new_t = Thread(target=self._fill_input_queue)
input_threads.append(new_t)
input_threads[-1].daemon = True
input_threads[-1].start()
self._input_threads = input_threads
bucketing_threads = []
for t in self._bucketing_threads:
if t.is_alive():
bucketing_threads.append(t)
else:
LOGGER.error('found bucketing thread dead')
new_t = Thread(target=self._fill_bucket_input_queue)
bucketing_threads.append(new_t)
bucketing_threads[-1].daemon = True
bucketing_threads[-1].start()
self._bucketing_threads = bucketing_threads
def __init__(self, dim1, dim2, initializer, expand=True):
"""The initializer can be an array, an object with [][]
accessor, a file path (string), a single floating point number
within [0,1] (the array is uniformly initialized to the same
value), or a user-provided callable that takes two integers x
and y in [0, dim1[ and [0, dim2[ respectively, and returns the
value to be stored in the array at [x][y]. The optional
parameter expand affects the case where the initializer is a
callable, an object with __getitem__, or a single number. In
those case, setting expand to False prevents the
precomputation of the whole array, and the InputSample
accessor encapsulate the function call, the object accessor,
or always returns the given number. If expand is True, the
InputSample created is mutable. If expand is False, the
InputSample is immutable."""
self._array = []
self._getitem = lambda k: self._array[k]
self._setitem = self._assign_to_array
if isinstance(initializer, basestring):
try:
self._array = read_input_data(initializer, dim1, dim2)
except IOError as e:
LOGGER.error("Could not read file %s.", initializer)
raise e
elif isinstance(initializer, types.FileType):
raise TypeError("Pass a string with the filepath to the "
"InputSample initializer, instead of a "
"file descriptor.")
elif isinstance(initializer, list):
self._array = initializer
elif hasattr(initializer, '__getitem__'):
if expand:
for x in xrange(dim1):
self._array.append([])
for y in xrange(dim2):
self._array[x].append(initializer[x][y])
else:
self._array = initializer
self._setitem = self._raise_immutable
elif hasattr(initializer, '__call__'):
# to restrict to functions:
# isinstance(initializer,
# (types.FunctionType, types.BuiltinFunctionType))
if expand:
for x in xrange(dim1):
self._array.append([])
for y in xrange(dim2):
self._array[x].append(initializer(x,y))
else:
class InitCont(object):
def __init__(self, x):
self._x = x
def __getitem__(self, y):
return initializer(self._x, y)
self._getitem = lambda x: InitCont(x)
self._setitem = self._raise_immutable
self._dim1 = dim1
self._dim2 = dim2
if expand:
verify_input_array(self._array, dim1, dim2)
def get_reference(self, avro_type):
if not avro_type in self.types.keys():
msg = 'No schema for type %s' % (avro_type)
LOGGER.error(msg)
raise KeyError(msg)
return self.types.get(avro_type).get_reference()
async def reply(client, message):
try:
inline = await client.get_inline_bot_results(Config.BOT_USERNAME,
"ETHO_ORUTHAN_PM_VANNU")
m = await client.send_inline_bot_result(message.chat.id,
query_id=inline.query_id,
result_id=inline.results[0].id,
hide_via=True)
old = Config.msg.get(message.chat.id)
if old:
await client.delete_messages(message.chat.id,
[old["msg"], old["s"]])
Config.msg[message.chat.id] = {
"msg": m.updates[1].message.id,
"s": message.message_id
}
except BotInlineDisabled:
LOGGER.error(
f"Error: Inline Mode for @{Config.BOT_USERNAME} is not enabled. Enable from @Botfather to enable PM Permit."
)
await message.reply(
f"{Config.REPLY_MESSAGE}\n\n<b>You can't use this bot in your group, for that you have to make your own bot from the [SOURCE CODE](https://github.com/subinps/VCPlayerBot) below.</b>",
disable_web_page_preview=True)
except Exception as e:
LOGGER.error(e, exc_info=True)
pass
def load_config(output_path):
config_file = os.path.join(output_path, 'config.ini')
config = configparser.ConfigParser()
if os.path.exists(config_file):
config.read(config_file)
else:
config.read_dict({
'DB': {
'host': 'localhost',
'user': '******',
'passwd': '1234',
'db': 'first_bot'
},
'Telegram': {
'API_TOKEN': 'API_Bot'
}
})
try:
with open(config_file, 'w', encoding='utf-8') as f:
config.write(f)
LOGGER.info(f'Конфигурация записана в {config_file}')
except Exception as e:
LOGGER.error(
f'Возникла ошибка при Записи данных: {e} в файл {config_file}')
LOGGER.debug(traceback.format_exc())
return config, config_file
def handle_read(self):
data = self.recv(BUF_SIZE)
if not data:
return
self.buffer_recv += data
#LOGGER.debug('%s local recv %s', id(self), data)
while True:
if self.stage == STAGE_INIT:
if len(self.buffer_recv) < 3:
return
ver = self.buffer_recv[0]
nmethod = self.buffer_recv[1]
method = self.buffer_recv[2]
if ver != 5 or nmethod != 1 or method != 0:
self.handle_close()
LOGGER.error("error init data: ver=%d, nmethod=%d, method=%d", ver, nmethod, method)
return
self.buffer_recv = self.buffer_recv[3:]
self.buffer_send += b'\x05\x00'
self.stage = STAGE_HANDSHAKE
continue
elif self.stage == STAGE_HANDSHAKE:
if len(self.buffer_recv) < 4:
return
ver = self.buffer_recv[0]
cmd = self.buffer_recv[1]
rsv = self.buffer_recv[2]
atyp = self.buffer_recv[3]
if ver != 5 or cmd != 1 or rsv != 0 or (atyp != 1 and atyp != 3):
self.handle_close()
LOGGER.error("error handshake data: ver=%d, cmd=%d, rsv=%d, atyp=%d", ver, cmd, rsv, atyp)
return
addr = ""
port = 0
if atyp == 1:
if len(self.buffer_recv) < 7:
return
addr = str(self.buffer_recv[4]) + '.' + str(self.buffer_recv[5]) + \
'.' + str(self.buffer_recv[6]) + '.' + str(self.buffer_recv[7])
port = self.buffer_recv[8] * 256 + self.buffer_recv[9]
self.buffer_send += b'\x05\x00\x00\x01' + self.buffer_recv[4:10]
elif atyp == 3:
alen = self.buffer_recv[4]
if len(self.buffer_recv) < 5 + alen + 2:
return
addr = self.buffer_recv[5:5 + alen].decode('utf-8')
port = self.buffer_recv[5 + alen] * 256 + self.buffer_recv[5 + alen + 1]
self.buffer_send += b'\x05\x00\x00\x03' + self.buffer_recv[4:5 + alen + 3]
LOGGER.info('%s local handshake: %s:%d', id(self), addr, port)
self.remote = RemoteConnection(self.server.server_addr, self.server.server_port, self.server.token)
self.remote.local = self
self.remote.buffer_send_raw += self.buffer_recv[3:] # include atyp + addr + port
self.buffer_recv = b''
self.stage = STAGE_STREAM
continue
elif self.stage == STAGE_STREAM:
self.remote.buffer_send_raw += self.buffer_recv
self.buffer_recv = b''
return
def main(unused_argv):
assert FLAGS.command
if FLAGS.command == 'build_vocab':
build_vocab()
elif FLAGS.command == 'build_binary':
convert_text_to_binary()
elif FLAGS.command == 'build_text':
convert_binary_to_text()
else:
LOGGER.error('not support command: %s', FLAGS.command)
def _gen_text(self, example):
while True:
ex = six.next(example)
try:
article_text = self._get_example_feature_text(ex, self._article_key)
abstract_text = self._get_example_feature_text(ex, self._abstract_key)
except ValueError:
LOGGER.error('failed to get article or abstract from example')
continue
yield (article_text, abstract_text)
def process_molo_tcp_pack(self):
"""Handle received TCP packet."""
ret = True
while ret:
ret = self.molo_tcp_pack.recv_buffer(self.append_recv_buffer)
if ret and self.molo_tcp_pack.error_code == MoloTcpPack.ERR_OK:
self.process_json_pack(self.molo_tcp_pack.body_jdata)
self.append_recv_buffer = self.molo_tcp_pack.tmp_buffer
if self.molo_tcp_pack.error_code == MoloTcpPack.ERR_MALFORMED:
LOGGER.error("tcp pack malformed!")
self.handle_close()
def handle_read(self):
"""Handle read message."""
buff = self.recv(BUFFER_SIZE)
if not buff:
return
remotesession = MOLO_CLIENT_APP.remote_session_dict.get(id(self))
if not remotesession:
LOGGER.error("LocalSession handle_read remove session not found")
self.handle_close()
return
LOGGER.debug("local session handle_read %s", buff)
remotesession.send_raw_pack(buff)
def add_traffic(self, token, size_mb):
if token not in self.traffic_map:
self.traffic_map[token] = 0
self.traffic_map[token] += size_mb
if self.last_traffic_save_time == None or time.time(
) - self.last_traffic_save_time > TRAFFIC_SAVE_INTERVAL:
self.last_traffic_save_time = time.time()
if save_traffic(self.traffic_map):
self.traffic_map = {}
else:
LOGGER.error('cannot save traffic file!')
async def stream(client, m: Message):
with suppress(MessageIdInvalid, MessageNotModified):
msg = await m.reply("Checking the recived input.")
if m.reply_to_message and m.reply_to_message.text:
link = m.reply_to_message.text
elif " " in m.text:
text = m.text.split(" ", 1)
link = text[1]
else:
k = await msg.edit("Provide a link to stream!")
await delete_messages([m, k])
return
regex = r"^(?:https?:\/\/)?(?:www\.)?youtu\.?be(?:\.com)?\/?.*(?:watch|embed)?(?:.*v=|v\/|\/)([\w\-_]+)\&?"
match = re.match(regex, link)
if match:
stream_link = await get_link(link)
if not stream_link:
k = await msg.edit("This is an invalid link.")
await delete_messages([m, k])
return
else:
stream_link = link
try:
is_audio_ = await is_audio(stream_link)
except:
is_audio_ = False
LOGGER.error("Unable to get Audio properties within time.")
if not is_audio_:
k = await msg.edit(
"This is an invalid link, provide me a direct link or a youtube link."
)
await delete_messages([m, k])
return
try:
dur = await get_duration(stream_link)
except:
dur = 0
if dur != 0:
k = await msg.edit("This is not a live stream, Use /play command.")
await delete_messages([m, k])
return
k, msg_ = await stream_from_link(stream_link)
if k == False:
k = await msg.edit(msg_)
await delete_messages([m, k])
return
if Config.msg.get('player'):
await Config.msg['player'].delete()
Config.msg['player'] = await msg.edit(
f"[Streaming]({stream_link}) Started. ㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤㅤ",
disable_web_page_preview=True,
reply_markup=await get_buttons())
await delete_messages([m])
def process_molo_tcp_pack(self):
"""Handle TCP packet."""
ret = self.molo_tcp_pack.recv_buffer(self.append_recv_buffer)
if ret and self.molo_tcp_pack.error_code == MoloTcpPack.ERR_OK:
self.append_recv_buffer = self.molo_tcp_pack.tmp_buffer
LOGGER.debug("RemoteSession process_molo_tcp_pack body:%s",
str(self.molo_tcp_pack.body_jdata))
self.process_json_pack(self.molo_tcp_pack.body_jdata)
if not self.tranparency:
if self.molo_tcp_pack.error_code == MoloTcpPack.ERR_MALFORMED:
LOGGER.error("tcp pack malformed!")
self.handle_close()
def handle_error(e):
"""
A catch-all error handler that catches any unhandled exception and returns
an error message to the end user.
"""
if isinstance(e, AuthError) or isinstance(e, InvalidUsage):
response = jsonify(e.to_dict())
response.status_code = e.code
else:
LOGGER.error('Internal Error: {}'.format(traceback.format_exc()))
response = jsonify(error_response('Internal Error',
'internal_error', 500))
response.status_code = 500
return response
def on_new_tunnel(self, jdata):
"""Handle on_new_tunnel json packet."""
LOGGER.debug("on_new_tunnel %s", str(jdata))
data = jdata['OnlineConfig']
if 'ping_interval' in jdata['OnlineConfig']:
MOLO_CLIENT_APP.ping_interval = jdata['OnlineConfig'][
'ping_interval']
self.update_notify_state(data)
if jdata['Payload']['Error'] != '':
LOGGER.error('Server failed to allocate tunnel: %s',
jdata['Payload']['Error'])
return
self.client_token = jdata['Payload']['token']
self.on_bind_status(jdata)
def get(self, _id):
"""
Return a single Task given an id.
:param _id: the id of the Task to return.
:return: a Task
"""
try:
response = self.service.find_by_id(_id)
respond_success(self, response, 200)
except MediaProError as error:
respond_error(self, error)
except Exception as e:
LOGGER.error(e, exc_info=True)
error = MediaProError(MediaProError.GENERIC)
respond_error(self, error)
def _db_url(self):
if MODE == 'prod':
return os.environ['DATABASE_URL']
proc = subprocess.run('heroku config:get DATABASE_URL -a scatbot',
capture_output=True,
shell=True,
text=True)
if proc.returncode != 0 or proc.stdout is None:
LOGGER.error('Failed to retrieve Heroku database URL. Aborting')
sys.exit(1)
else:
LOGGER.info('Successfully retrieved Heroku database URL')
return proc.stdout.strip()
def recv_body(self):
"""Read received TCP body."""
if len(self.tmp_buffer) < self.body_len:
return False
try:
json_buff = self.tmp_buffer[:self.body_len].decode('utf-8')
self.body_jdata = json.loads(json_buff)
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
self.error_code = MoloTcpPack.ERR_MALFORMED
LOGGER.error("MoloTcpPack recv body error %s",
self.tmp_buffer[:self.body_len])
logging.exception(exc)
return False
self.tmp_buffer = self.tmp_buffer[self.body_len:]
return True
def delete(self, _id, key):
"""
Deletes a certain first level key from the payload if it exists.
:param _id: the id of the Task to delete its payload
:param key: the key to delete
:return: the modified Task
"""
try:
task = self.service.remove_payload_key_by_task_id(_id, key)
respond_success(self, task)
except MediaProError as error:
respond_error(self, error)
except Exception as e:
LOGGER.error(e, exc_info=True)
error = MediaProError(MediaProError.GENERIC)
respond_error(self, error)
def recv_header_prefix(self):
"""Read received TCP header prefix."""
if len(self.tmp_buffer) < MoloTcpPack.HEADER_PREFIX_EN:
return False
self.magic = self.tmp_buffer[:MoloTcpPack.MAGIC_LEN]
if self.magic != MoloTcpPack.MOLO_TCP_MAGIC:
self.error_code = MoloTcpPack.ERR_MALFORMED
LOGGER.error("wrong tcp header magic %s", self.magic)
return False
self.header_len = bytetolen(
self.tmp_buffer[MoloTcpPack.MAGIC_LEN:MoloTcpPack.
HEADER_PREFIX_EN])
self.tmp_buffer = self.tmp_buffer[MoloTcpPack.HEADER_PREFIX_EN:]
return True
def patch(self, _id):
"""
Updates a Task given one or more fields.
:param _id: the id of the Taks to update.
:return: the Task updated.
"""
try:
task = JSONDecoder().decode(self.request.body)
self.service.update_by_id(_id, task)
respond_success(self, task)
except MediaProError as error:
respond_error(self, error)
except Exception as e:
LOGGER.error(e, exc_info=True)
error = MediaProError(MediaProError.GENERIC)
respond_error(self, error)
def post(self):
"""
Creates a new Task.
:return: the Task created.
"""
try:
task = JSONDecoder().decode(self.request.body)
task_id = self.service.insert(task).inserted_id
task['_id'] = task_id
respond_success(self, task)
except MediaProError as error:
respond_error(self, error)
except Exception as e:
LOGGER.error(e, exc_info=True)
error = MediaProError(MediaProError.GENERIC)
respond_error(self, error)
def post(self, _id):
"""
Adds or replace payload first level key, and returns the modified task
:param _id: the id of the Task to update its payload.
:return: the modified Task
"""
try:
payload = JSONDecoder().decode(self.request.body)
task = self.service.update_payload_by_task_id(_id, payload)
respond_success(self, task)
except MediaProError as error:
respond_error(self, error)
except Exception as e:
LOGGER.error(e, exc_info=True)
error = MediaProError(MediaProError.GENERIC)
respond_error(self, error)
def load_settings():
try:
config, config_file = load_config(EXE_PATH)
dict_setting['host'] = config.get('DB', 'host')
dict_setting['user'] = config.get('DB', 'user')
dict_setting['passwd'] = config.get('DB', 'passwd')
dict_setting['db'] = config.get('DB', 'db')
if config.get('Telegram', 'API_TOKEN') == 'API_Bot':
LOGGER.info(f'Замените данные в файле {config_file} на реальные!')
return False
else:
dict_setting['api_token'] = config.get('Telegram', 'API_TOKEN')
except Exception as e:
LOGGER.error(f'Возникла ошибка при Загрузки данных: {e}')
LOGGER.debug(traceback.format_exc())
return dict_setting
def post(self, action):
try:
# Fetch appropriate handler
if not hasattr(self, str(action)):
raise RouteNotFound(action)
# Pass along the data and get a result
handler = getattr(self, str(action))
handler(self.request.body)
except CustomError as e:
self.respond(e.message, e.code)
except Exception as e:
LOGGER.error(
"\n\n======== SGMomPassChecker SERVER ERROR ========\n%s\n%s\n",
__file__,
traceback.format_exc()
)
error = ServerError()
self.respond(error.message, error.code)
def build_vocab():
"""build vocab from raw data in text format.
input data format:
each line looks like:
article=<d> <p> <s> word1 word2 ... </s> <s> ... </s> </p> ... </d>\tabstract=<d> <p> <s> ... </s> </p> ... </d>
"""
data_path = FLAGS.in_file
vocab_path = FLAGS.out_file
assert data_path and vocab_path, 'filename of data and vocabulary should be provided'
if not gfile.Exists(vocab_path):
LOGGER.debug('build vocabulary from %s, storing it into %s', data_path, vocab_path)
vocab = {}
counter = 0
reader = codecs.open(data_path, mode='rb', encoding='utf-8')
for line in reader:
counter += 1
if counter % 1000 == 0:
LOGGER.debug("processing line %d", counter)
for feature in line.strip().split(FLAGS.feature_separator):
(k, v) = feature.split('=')
word_freq = {k: len(list(g)) for k, g in groupby(sorted(v.split())) if k not in SPECIAL_TOKENS}
for word, freq in word_freq.items():
vocab[word] = vocab.get(word, 0) + freq
reader.close()
vocab = sorted(vocab.iteritems(), key=lambda kv: kv[1], reverse=True)
vocab = [(k, v) for k, v in SPECIAL_TOKENS_FREQ.items()] + vocab
with gfile.GFile(vocab_path, mode='wb') as vocab_file:
for word, freq in vocab:
vocab_file.write(word + b'\t' + str(freq) + b'\n')
else:
LOGGER.error('vocabulary file exist: %s', vocab_path)
def register(self, name, payload=None):
# register an entity of type 'name'
# if no payload is passed, an appropriate one will be created
count = self.type_count.get(name, 0)
count += 1
self.type_count[name] = count
if not payload:
payload = self.types[name].get()
# type_name = self.alias.get(name)
type_id = self.schema_id.get(name)
ps_id = self.schema_decorator.get(type_id)
data = self.payload_to_data(ps_id, payload)
try:
self.client.entities.create(data=data)
LOGGER.debug('Created instance # %s of type %s' % (self.type_count[name], name))
except AetherAPIException as err:
LOGGER.error('in creation of entity of type %s: %s' % (name, err))
return data
async def add_admin(client, message):
if message.reply_to_message:
if message.reply_to_message.from_user.id is None:
k = await message.reply("You are an anonymous admin, you can't do this.")
await delete_messages([message, k])
return
user_id=message.reply_to_message.from_user.id
user=message.reply_to_message.from_user
elif ' ' in message.text:
c, user = message.text.split(" ", 1)
if user.startswith("@"):
user=user.replace("@", "")
try:
user=await client.get_users(user)
except Exception as e:
k=await message.reply(f"I was unable to locate that user.\nError: {e}")
LOGGER.error(f"Unable to find the user - {e}", exc_info=True)
await delete_messages([message, k])
return
user_id=user.id
else:
try:
user_id=int(user)
user=await client.get_users(user_id)
except:
k=await message.reply(f"You should give a user id or his username with @.")
await delete_messages([message, k])
return
else:
k=await message.reply("No user specified, reply to a user with /vcpromote or pass a users user id or username.")
await delete_messages([message, k])
return
if user_id in Config.ADMINS:
k = await message.reply("This user is already an admin.")
await delete_messages([message, k])
return
Config.ADMINS.append(user_id)
k=await message.reply(f"Succesfully promoted {user.mention} as VC admin")
await sync_to_db()
await delete_messages([message, k])
def convert_binary_to_text():
"""convert binary data to text
output data format:
each line looks like:
article=<d> <p> <s> word1 word2 ... </s> <s> ... </s> </p> ... </d>\tabstract=<d> <p> <s> ... </s> </p> ... </d>
"""
binary_data_path = FLAGS.in_file
text_data_path = FLAGS.out_file
assert binary_data_path and text_data_path, 'filename of binary data or text data should be provided'
if not gfile.Exists(text_data_path):
LOGGER.debug('convert binary to text format: %s => %s', binary_data_path, text_data_path)
reader = open(binary_data_path, mode='rb')
writer = codecs.open(text_data_path, mode='wb', encoding='utf-8')
while True:
len_bytes = reader.read(8)
if not len_bytes:
LOGGER.debug('done reading')
break
str_len = struct.unpack('q', len_bytes)[0]
tf_example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
tf_example = example_pb2.Example.FromString(tf_example_str)
examples = []
for key in tf_example.features.feature:
value = tf_example.features.feature[key].bytes_list.value[0]
value = value.decode('utf-8')
examples.append('%s=%s' % (key, value))
writer.write('%s\n' % FLAGS.feature_separator.join(examples))
writer.close()
reader.close()
else:
LOGGER.error('text data exist: %s', text_data_path)
def delete(self, _id):
"""
Deletes a Task.
:param _id: the id of the Task to delete.
:return: a message if the deletion was successfully or not.
"""
try:
result = self.service.delete_by_id(_id)
response = {
'message': "The task with id: {0} has been deleted.".format(
_id)
}
if result.deleted_count == 0:
response['message'] = "No task has been deleted. This could "\
"due there is no task with such id or "\
"the task has a RUNNING status."
respond_success(self, response, 200)
except Exception as e:
LOGGER.error(e, exc_info=True)
error = MediaProError(MediaProError.GENERIC)
respond_error(self, error)
def _eval(model, config, batch_reader, vocab=None):
model.build_graph()
saver = tf.train.Saver()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir)
sess = tf.Session(config=config)
running_avg_loss = 0
step = 0
while True:
time.sleep(FLAGS.eval_interval_secs)
try:
ckpt_state = tf.train.get_checkpoint_state(FLAGS.log_root)
except tf.errors.OutOfRangeError as e:
LOGGER.error('cannot restore checkpoint: %s', e)
continue
if not (ckpt_state and ckpt_state.model_checkpoint_path):
LOGGER.info('no model to eval yet at %s', FLAGS.train_dir)
continue
LOGGER.info('loading checkpoint %s', ckpt_state.model_checkpoint_path)
saver.restore(sess, ckpt_state.model_checkpoint_path)
(
article_batch, abstract_batch, targets, article_lens, abstract_lens, loss_weights, _, _
) = batch_reader.next_batch()
(summaries, loss, train_step) = model.eval(
sess, config, article_batch, abstract_batch, targets, article_lens, abstract_lens, loss_weights
)
LOGGER.info('article: %s', ' '.join(data.convert_ids_to_words(article_batch[0][:].tolist(), vocab)))
LOGGER.info('abstract: %s', ' '.join(data.convert_ids_to_words(abstract_batch[0][:].tolist(), vocab)))
summary_writer.add_summary(summaries, train_step)
running_avg_loss = _running_avg_loss(running_avg_loss, loss, summary_writer, train_step)
if step % 100 == 0:
summary_writer.flush()
def register():
project = register_project()
project_id = project.id
if not project_id:
LOGGER.error('project could not be registerd, does it already exist?')
sys.exit(0)
schema_info = schema()
if not schema_info:
raise ValueError('No schemas registered')
LOGGER.debug(schema_info)
schema_ids = {obj.name: obj.id for obj in schema_info}
LOGGER.debug(schema_ids)
schema_decorators = schema_decorator(project_id, schema_ids)
LOGGER.debug(schema_decorators)
ps_ids = {ps.name: ps.id for ps in schema_decorators.values()}
ms = mappingset(project_id)
sub_id = mapping(project_id, ms.id, ps_ids)
LOGGER.debug(sub_id)
def proxy_loop(self):
"""Handle main loop and reconnection."""
self.molo_client.sock_connect()
while not self.is_exited:
try:
asyncore.loop(map=self.async_map)
except asyncore.ExitNow as exc:
logging.exception(exc)
LOGGER.error("asyncore.loop exception")
if not self.is_exited:
try:
asyncore.close_all()
self.molo_client.sock_connect()
time.sleep(RECONNECT_INTERVAL)
LOGGER.info("moloserver reconnecting...")
except Exception as exc:
print("proxy_loop(): " + str(exc))
LOGGER.info("reconnect failed, retry...")
time.sleep(RECONNECT_INTERVAL)
asyncore.close_all()
LOGGER.debug("proxy exited")
def get(self):
"""
Returns all tasks using pagination and filtering.
Example of usage:
/api/tasks?to=1469930541448&status=SCHEDULED&worker=task_worker_001
Valid filter params are:
page: int
rows: int
worker: str
status: str
from: int (timestamp)
to: int (timestamp)
:return: all tasks using pagination.
"""
try:
response = [doc for doc in self.service.find(self)]
respond_success(self, response)
except MediaProError as error:
respond_error(self, error)
except Exception as e:
LOGGER.error(e, exc_info=True)
error = MediaProError(MediaProError.GENERIC)
respond_error(self, error)
def schema():
schema_names = []
schemas = []
defs = file_to_json(f'{RESOURCES_DIR}/schemas/all.json')
for obj in defs:
name = obj.get('name')
schema_names.append(name)
schema_obj = {
'name': name,
'type': 'record',
'revision': '1',
'definition': obj
}
schemas.append(schema_obj)
out = []
for obj in schemas:
try:
out.append(client.schemas.create(data=obj))
except Exception as err:
LOGGER.error(err)
LOGGER.error(obj)
return out
def main(seed_size=1000):
SEED_ENTITIES = seed_size
manager = None
building = 'eha.aether.clusterdemo.Building'
person = 'eha.aether.clusterdemo.Person'
manager = MockingManager(KERNEL_URL, KERNEL_USER, KERNEL_PASSWORD,
CLIENT_LOGLEVEL, REALM, KEYCLOAK_URL)
for i in manager.types.keys():
LOGGER.error(i)
for k, v in manager.names.items():
LOGGER.error([k, v])
try:
manager.types[building].override_property('latitude',
MockFn(LOCATION.get_lat))
except KeyError:
LOGGER.error(
'%s is not a valid registered type. Have you run scripts/register_assets.sh?'
% building)
sys.exit(1)
manager.types[building].override_property('longitude',
MockFn(LOCATION.get_lng))
manager.types[building].override_property('altitude',
MockFn(LOCATION.get_alt))
manager.types[building].override_property('accuracy',
MockFn(LOCATION.get_acc))
manager.types[building].override_property('building_type',
MockFn(LOCATION.get_btype))
manager.types[person].override_property('occupant_age',
MockFn(PERSON.get_age))
manager.types[person].override_property('occupant_gender',
MockFn(PERSON.get_sex))
manager.types[person].override_property('occupant_name',
MockFn(PERSON.get_name))
for _ in range(SEED_ENTITIES):
manager.register(person)
for mocker in manager.types.values():
if mocker.killed:
manager.kill()
return
break
manager.kill()
def run_commands_on_ec2_instance(ec2_connection, is_gpu):
"""
This function assumes that the required 'serve' folder is already available on the ec2 instance in the home directory.
Returns a map of the command executed and return value of that command.
"""
command_result_map = {}
virtual_env_name = "venv"
with ec2_connection.cd(f"/home/ubuntu/serve"):
ec2_connection.run(f"python3 -m venv {virtual_env_name}")
with ec2_connection.prefix(f"source {virtual_env_name}/bin/activate"):
commands_list = GPU_INSTANCE_COMMANDS_LIST if is_gpu else CPU_INSTANCE_COMMANDS_LIST
for command in commands_list:
LOGGER.info(
f"*** Executing command on ec2 instance: {command}")
ret_obj = ec2_connection.run(
command,
echo=True,
warn=True,
shell="/bin/bash",
env={
"LC_CTYPE": "en_US.utf8",
"JAVA_HOME": "/usr/lib/jvm/java-11-openjdk-amd64",
"PYTHONIOENCODING": "utf8",
},
encoding="utf8")
if ret_obj.return_code != 0:
LOGGER.error(f"*** Failed command: {command}")
LOGGER.error(
f"*** Failed command stdout: {ret_obj.stdout}")
LOGGER.error(
f"*** Failed command stderr: {ret_obj.stderr}")
command_result_map[command] = ret_obj.return_code
return command_result_map
def _request(self, method, arguments=None, ids=None, require_ids=False, timeout=None):
"""
Send json-rpc request to Transmission using http POST
"""
if not isinstance(method, string_types):
raise ValueError('request takes method as string')
if arguments is None:
arguments = {}
if not isinstance(arguments, dict):
raise ValueError('request takes arguments as dict')
ids = parse_torrent_ids(ids)
if len(ids) > 0:
arguments['ids'] = ids
elif require_ids:
raise ValueError('request require ids')
query = json.dumps({'tag': self._sequence, 'method': method
, 'arguments': arguments})
self._sequence += 1
start = time.time()
http_data = self._http_query(query, timeout)
elapsed = time.time() - start
LOGGER.info('http request took %.3f s' % (elapsed))
try:
data = json.loads(http_data)
except ValueError as error:
LOGGER.error('Error: ' + str(error))
LOGGER.error('Request: \"%s\"' % (query))
LOGGER.error('HTTP data: \"%s\"' % (http_data))
raise
LOGGER.debug(json.dumps(data, indent=2))
if 'result' in data:
if data['result'] != 'success':
raise TransmissionError('Query failed with result \"%s\".' % (data['result']))
else:
raise TransmissionError('Query failed without result.')
results = {}
if method == 'torrent-get':
for item in data['arguments']['torrents']:
results[item['id']] = Torrent(self, item)
if self.protocol_version == 2 and 'peers' not in item:
self.protocol_version = 1
elif method == 'torrent-add':
item = None
if 'torrent-added' in data['arguments']:
item = data['arguments']['torrent-added']
elif 'torrent-duplicate' in data['arguments']:
item = data['arguments']['torrent-duplicate']
if item:
results[item['id']] = Torrent(self, item)
else:
raise TransmissionError('Invalid torrent-add response.')
elif method == 'session-get':
self._update_session(data['arguments'])
elif method == 'session-stats':
# older versions of T has the return data in "session-stats"
if 'session-stats' in data['arguments']:
self._update_session(data['arguments']['session-stats'])
else:
self._update_session(data['arguments'])
elif method in ('port-test', 'blocklist-update', 'free-space', 'torrent-rename-path'):
results = data['arguments']
else:
return None
return results
async def add_to_playlist(_, message: Message):
with suppress(MessageIdInvalid, MessageNotModified):
admins = await get_admins(Config.CHAT)
if Config.ADMIN_ONLY:
if not (message.from_user is None and message.sender_chat
or message.from_user.id in admins):
k = await message.reply_sticker(
"CAADBQADsQIAAtILIVYld1n74e3JuQI")
await delete_messages([message, k])
return
type = ""
yturl = ""
ysearch = ""
url = ""
if message.command[0] == "fplay":
if not (message.from_user is None and message.sender_chat
or message.from_user.id in admins):
k = await message.reply("This command is only for admins.")
await delete_messages([message, k])
return
msg = await message.reply_text("⚡️ **Checking recived input..**")
if message.reply_to_message and message.reply_to_message.video:
await msg.edit("⚡️ **Checking Telegram Media...**")
type = 'video'
m_video = message.reply_to_message.video
elif message.reply_to_message and message.reply_to_message.document:
await msg.edit("⚡️ **Checking Telegram Media...**")
m_video = message.reply_to_message.document
type = 'video'
if not "video" in m_video.mime_type:
return await msg.edit("The given file is invalid")
elif message.reply_to_message and message.reply_to_message.audio:
#if not Config.IS_VIDEO:
#return await message.reply("Play from audio file is available only if Video Mode if turned off.\nUse /settings to configure ypur player.")
await msg.edit("⚡️ **Checking Telegram Media...**")
type = 'audio'
m_video = message.reply_to_message.audio
else:
if message.reply_to_message and message.reply_to_message.text:
query = message.reply_to_message.text
elif " " in message.text:
text = message.text.split(" ", 1)
query = text[1]
else:
await msg.edit(
"You Didn't gave me anything to play.Reply to a video or a youtube link or a direct link."
)
await delete_messages([message, msg])
return
regex = r"^(?:https?:\/\/)?(?:www\.)?youtu\.?be(?:\.com)?\/?.*(?:watch|embed)?(?:.*v=|v\/|\/)([\w\-_]+)\&?"
match = re.match(regex, query)
if match:
type = "youtube"
yturl = query
elif query.startswith("http"):
try:
has_audio_ = await is_audio(query)
except:
has_audio_ = False
LOGGER.error("Unable to get Audio properties within time.")
if has_audio_:
try:
dur = await get_duration(query)
except:
dur = 0
if dur == 0:
await msg.edit(
"This is a live stream, Use /stream command.")
await delete_messages([message, msg])
return
type = "direct"
url = query
else:
if is_ytdl_supported(query):
type = "ytdl_s"
url = query
else:
await msg.edit(
"This is an invalid link, provide me a direct link or a youtube link."
)
await delete_messages([message, msg])
return
else:
type = "query"
ysearch = query
if not message.from_user is None:
user = f"[{message.from_user.first_name}](tg://user?id={message.from_user.id})"
user_id = message.from_user.id
else:
user = "******"
user_id = "anonymous_admin"
now = datetime.now()
nyav = now.strftime("%d-%m-%Y-%H:%M:%S")
if type in ["video", "audio"]:
if type == "audio":
if m_video.title is None:
if m_video.file_name is None:
title_ = "Music"
else:
title_ = m_video.file_name
else:
title_ = m_video.title
if m_video.performer is not None:
title = f"{m_video.performer} - {title_}"
else:
title = title_
unique = f"{nyav}_{m_video.file_size}_audio"
else:
title = m_video.file_name
unique = f"{nyav}_{m_video.file_size}_video"
if Config.PTN:
ny = parse(title)
title_ = ny.get("title")
if title_:
title = title_
file_id = m_video.file_id
if title is None:
title = 'Music'
data = {1: title, 2: file_id, 3: "telegram", 4: user, 5: unique}
if message.command[0] == "fplay":
pla = [data] + Config.playlist
Config.playlist = pla
else:
Config.playlist.append(data)
await add_to_db_playlist(data)
await msg.edit("Media added to playlist")
elif type in ["youtube", "query", "ytdl_s"]:
if type == "youtube":
await msg.edit("⚡️ **Fetching Video From YouTube...**")
url = yturl
elif type == "query":
try:
await msg.edit("⚡️ **Fetching Video From YouTube...**")
ytquery = ysearch
results = YoutubeSearch(ytquery, max_results=1).to_dict()
url = f"https://youtube.com{results[0]['url_suffix']}"
title = results[0]["title"][:40]
except Exception as e:
await msg.edit("Song not found.\nTry inline mode..")
LOGGER.error(str(e), exc_info=True)
await delete_messages([message, msg])
return
elif type == "ytdl_s":
url = url
else:
return
ydl_opts = {
"quite": True,
"geo-bypass": True,
"nocheckcertificate": True
}
ydl = YoutubeDL(ydl_opts)
try:
info = ydl.extract_info(url, False)
except Exception as e:
LOGGER.error(e, exc_info=True)
await msg.edit(f"YouTube Download Error ❌\nError:- {e}")
LOGGER.error(str(e))
await delete_messages([message, msg])
return
if type == "ytdl_s":
title = "Music"
try:
title = info['title']
except:
pass
else:
title = info["title"]
if info['duration'] is None:
await msg.edit(
"This is a live stream, Use /stream command.")
await delete_messages([message, msg])
return
data = {
1: title,
2: url,
3: "youtube",
4: user,
5: f"{nyav}_{user_id}"
}
if message.command[0] == "fplay":
pla = [data] + Config.playlist
Config.playlist = pla
else:
Config.playlist.append(data)
await add_to_db_playlist(data)
await msg.edit(f"[{title}]({url}) added to playist",
disable_web_page_preview=True)
elif type == "direct":
data = {
1: "Music",
2: url,
3: "url",
4: user,
5: f"{nyav}_{user_id}"
}
if message.command[0] == "fplay":
pla = [data] + Config.playlist
Config.playlist = pla
else:
Config.playlist.append(data)
await add_to_db_playlist(data)
await msg.edit("Link added to playlist")
if not Config.CALL_STATUS \
and len(Config.playlist) >= 1:
await msg.edit("Downloading and Processing...")
await download(Config.playlist[0], msg)
await play()
elif (len(Config.playlist) == 1 and Config.CALL_STATUS):
await msg.edit("Downloading and Processing...")
await download(Config.playlist[0], msg)
await play()
elif message.command[0] == "fplay":
await msg.edit("Downloading and Processing...")
await download(Config.playlist[0], msg)
await play()
else:
await send_playlist()
await msg.delete()
pl = await get_playlist_str()
if message.chat.type == "private":
await message.reply(pl,
reply_markup=await get_buttons(),
disable_web_page_preview=True)
elif not Config.LOG_GROUP and message.chat.type == "supergroup":
if Config.msg.get('playlist') is not None:
await Config.msg['playlist'].delete()
Config.msg['playlist'] = await message.reply(
pl,
disable_web_page_preview=True,
reply_markup=await get_buttons())
await delete_messages([message])
for track in Config.playlist[:2]:
await download(track)
from src.do_on_fail import do_on_fail
def run() -> None:
"""
Starting point of ETL
Here we execute ETL steps one-by-one
:return:
"""
LOGGER.info('starting etl')
LOGGER.info('step 1. specific script #1')
first_etl_pipeline()
LOGGER.info('step 2. specific script #2')
second_etl_pipeline()
LOGGER.info('etl completed')
if __name__ == '__main__':
try:
run()
except Exception as etl_exception:
# if any exception occurred during ETL - write error to file
# or send notification if possible (for that create special function in /utils)
with open(f"{ENV}_error_{datetime.now():%Y%m%d-%H%M}.log", "w") as f:
f.write(str(etl_exception))
f.write(traceback.format_exc())
LOGGER.info("doing something on ETL fail")
do_on_fail()
LOGGER.error("etl completed with errors")
raise etl_exception
def launch_ec2_instance(region, instance_type, ami_id):
"""
Note: This function relies on CODEBUILD environment variables. If this function is used outside of CODEBUILD,
modify the function accordingly.
Spins up an ec2 instance, clones the current Github Pull Request commit id on the instance, and runs sanity test on it.
Prints the output of the command executed.
"""
github_repo = os.environ.get(
"CODEBUILD_SOURCE_REPO_URL",
"https://github.com/pytorch/serve.git").strip()
github_pr_commit_id = os.environ.get("CODEBUILD_RESOLVED_SOURCE_VERSION",
"HEAD").strip()
github_hookshot = os.environ.get("CODEBUILD_SOURCE_VERSION",
"job-local").strip()
github_hookshot = github_hookshot.replace("/", "-")
# Extract the PR number or use the last 6 characters of the commit id
github_pull_request_number = github_hookshot.split(
"-")[1] if "-" in github_hookshot else github_hookshot[-6:]
ec2_client = boto3.client("ec2",
config=Config(retries={"max_attempts": 10}),
region_name=region)
random.seed(f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}")
ec2_key_name = f"{github_hookshot}-ec2-instance-{random.randint(1, 1000)}"
# Spin up ec2 instance and run tests
try:
key_file = ec2_utils.generate_ssh_keypair(ec2_client, ec2_key_name)
instance_details = ec2_utils.launch_instance(
ami_id,
instance_type,
ec2_key_name=ec2_key_name,
region=region,
user_data=None,
iam_instance_profile_name=ec2_utils.EC2_INSTANCE_ROLE_NAME,
instance_name=ec2_key_name,
)
instance_id = instance_details["InstanceId"]
ip_address = ec2_utils.get_public_ip(instance_id, region=region)
LOGGER.info(f"*** Waiting on instance checks to complete...")
ec2_utils.check_instance_state(instance_id,
state="running",
region=region)
ec2_utils.check_system_state(instance_id,
system_status="ok",
instance_status="ok",
region=region)
LOGGER.info(
f"*** Instance checks complete. Running commands on instance.")
# Create a fabric connection to the ec2 instance.
ec2_connection = ec2_utils.get_ec2_fabric_connection(
instance_id, key_file, region)
LOGGER.info(f"Running update command. This could take a while.")
ec2_connection.run(f"sudo apt update")
# Update command takes a while to run, and should ideally run uninterrupted
time.sleep(300)
with ec2_connection.cd("/home/ubuntu"):
LOGGER.info(
f"*** Cloning the PR related to {github_hookshot} on the ec2 instance."
)
ec2_connection.run(f"git clone {github_repo}")
ec2_connection.run(
f"cd serve && git fetch origin pull/{github_pull_request_number}/head:pull && git checkout pull"
)
ec2_connection.run(f"sudo apt-get install -y python3-venv")
# Following is necessary on Base Ubuntu DLAMI because the default python is python2
# This will NOT fail for other AMI where default python is python3
ec2_connection.run(
f"sudo cp /usr/local/bin/pip3 /usr/local/bin/pip && pip install --upgrade pip",
warn=True)
ec2_connection.run(
f"sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1",
warn=True)
is_gpu = True if instance_type[:2] in GPU_INSTANCES else False
command_return_value_map = run_commands_on_ec2_instance(
ec2_connection, is_gpu)
if any(command_return_value_map.values()):
raise ValueError(
f"*** One of the commands executed on ec2 returned a non-zero value."
)
else:
LOGGER.info(
f"*** All commands executed successfully on ec2. command:return_value map is as follows:"
)
LOGGER.info(command_return_value_map)
except ValueError as e:
LOGGER.error(f"*** ValueError: {e}")
LOGGER.error(
f"*** Following commands had the corresponding return value:")
LOGGER.error(command_return_value_map)
raise e
except Exception as e:
LOGGER.error(f"*** Exception occured. {e}")
raise e
finally:
LOGGER.warning(
f"*** Terminating instance-id: {instance_id} with name: {ec2_key_name}"
)
ec2_utils.terminate_instance(instance_id, region)
LOGGER.warning(f"*** Destroying ssh key_pair: {ec2_key_name}")
ec2_utils.destroy_ssh_keypair(ec2_client, ec2_key_name)
async def schedule_vc(bot, message):
with suppress(MessageIdInvalid, MessageNotModified):
type=""
yturl=""
ysearch=""
msg = await message.reply_text("⚡️ **Checking recived input..**")
if message.reply_to_message and message.reply_to_message.video:
await msg.edit("⚡️ **Checking Telegram Media...**")
type='video'
m_video = message.reply_to_message.video
elif message.reply_to_message and message.reply_to_message.document:
await msg.edit("⚡️ **Checking Telegram Media...**")
m_video = message.reply_to_message.document
type='video'
if not "video" in m_video.mime_type:
return await msg.edit("The given file is invalid")
elif message.reply_to_message and message.reply_to_message.audio:
#if not Config.IS_VIDEO:
#return await message.reply("Play from audio file is available only if Video Mode if turned off.\nUse /settings to configure ypur player.")
await msg.edit("⚡️ **Checking Telegram Media...**")
type='audio'
m_video = message.reply_to_message.audio
else:
if message.reply_to_message and message.reply_to_message.text:
query=message.reply_to_message.text
elif " " in message.text:
text = message.text.split(" ", 1)
query = text[1]
else:
await msg.edit("You Didn't gave me anything to schedule. Reply to a video or a youtube link or a direct link.")
await delete_messages([message, msg])
return
regex = r"^(?:https?:\/\/)?(?:www\.)?youtu\.?be(?:\.com)?\/?.*(?:watch|embed)?(?:.*v=|v\/|\/)([\w\-_]+)\&?"
match = re.match(regex,query)
if match:
type="youtube"
yturl=query
elif query.startswith("http"):
has_audio_ = await is_audio(query)
if not has_audio_:
if is_ytdl_supported(query):
type="ytdl_s"
url=query
else:
await msg.edit("This is an invalid link, provide me a direct link or a youtube link.")
await delete_messages([message, msg])
return
type="direct"
url=query
else:
type="query"
ysearch=query
if not message.from_user is None:
user=f"[{message.from_user.first_name}](tg://user?id={message.from_user.id}) - (Scheduled)"
user_id = message.from_user.id
else:
user="******"
user_id = "anonymous_admin"
now = datetime.now()
nyav = now.strftime("%d-%m-%Y-%H:%M:%S")
if type in ["video", "audio"]:
if type == "audio":
if m_video.title is None:
if m_video.file_name is None:
title_ = "Music"
else:
title_ = m_video.file_name
else:
title_ = m_video.title
if m_video.performer is not None:
title = f"{m_video.performer} - {title_}"
else:
title=title_
unique = f"{nyav}_{m_video.file_size}_audio"
else:
title=m_video.file_name
unique = f"{nyav}_{m_video.file_size}_video"
if Config.PTN:
ny = parse(title)
title_ = ny.get("title")
if title_:
title = title_
if title is None:
title = 'Music'
data={'1':title, '2':m_video.file_id, '3':"telegram", '4':user, '5':unique}
sid=f"{message.chat.id}_{msg.message_id}"
Config.SCHEDULED_STREAM[sid] = data
await sync_to_db()
elif type in ["youtube", "query", "ytdl_s"]:
if type=="youtube":
await msg.edit("⚡️ **Fetching Video From YouTube...**")
url=yturl
elif type=="query":
try:
await msg.edit("⚡️ **Fetching Video From YouTube...**")
ytquery=ysearch
results = YoutubeSearch(ytquery, max_results=1).to_dict()
url = f"https://youtube.com{results[0]['url_suffix']}"
title = results[0]["title"][:40]
except Exception as e:
await msg.edit(
"Song not found.\nTry inline mode.."
)
LOGGER.error(str(e), exc_info=True)
await delete_messages([message, msg])
return
elif type == "ytdl_s":
url=url
else:
return
ydl_opts = {
"quite": True,
"geo-bypass": True,
"nocheckcertificate": True
}
ydl = YoutubeDL(ydl_opts)
try:
info = ydl.extract_info(url, False)
except Exception as e:
LOGGER.error(e, exc_info=True)
await msg.edit(
f"YouTube Download Error ❌\nError:- {e}"
)
LOGGER.error(str(e))
await delete_messages([message, msg])
return
if type == "ytdl_s":
title = "Music"
try:
title=info['title']
except:
pass
else:
title = info["title"]
data={'1':title, '2':url, '3':"youtube", '4':user, '5':f"{nyav}_{user_id}"}
sid=f"{message.chat.id}_{msg.message_id}"
Config.SCHEDULED_STREAM[sid] = data
await sync_to_db()
elif type == "direct":
data={"1":"Music", '2':url, '3':"url", '4':user, '5':f"{nyav}_{user_id}"}
sid=f"{message.chat.id}_{msg.message_id}"
Config.SCHEDULED_STREAM[sid] = data
await sync_to_db()
if message.chat.type!='private' and message.from_user is None:
await msg.edit(
text="You cant schedule from here since you are an anonymous admin. Click the schedule button to schedule through private chat.",
reply_markup=InlineKeyboardMarkup(
[
[
InlineKeyboardButton(f"Schedule", url=f"https://telegram.dog/{Config.BOT_USERNAME}?start=sch_{sid}"),
]
]
),)
await delete_messages([message, msg])
return
today = datetime.now(IST)
smonth=today.strftime("%B")
obj = calendar.Calendar()
thisday = today.day
year = today.year
month = today.month
m=obj.monthdayscalendar(year, month)
button=[]
button.append([InlineKeyboardButton(text=f"{str(smonth)} {str(year)}",callback_data=f"sch_month_choose_none_none")])
days=["Mon", "Tues", "Wed", "Thu", "Fri", "Sat", "Sun"]
f=[]
for day in days:
f.append(InlineKeyboardButton(text=f"{day}",callback_data=f"day_info_none"))
button.append(f)
for one in m:
f=[]
for d in one:
year_=year
if d < int(today.day):
year_ += 1
if d == 0:
k="\u2063"
d="none"
else:
k=d
f.append(InlineKeyboardButton(text=f"{k}",callback_data=f"sch_month_{year_}_{month}_{d}"))
button.append(f)
button.append([InlineKeyboardButton("Close", callback_data="schclose")])
await msg.edit(f"Choose the day of the month you want to schedule the voicechat.\nToday is {thisday} {smonth} {year}. Chooosing a date preceeding today will be considered as next year {year+1}", reply_markup=InlineKeyboardMarkup(button))
async def channel_play_list(client, m: Message):
with suppress(MessageIdInvalid, MessageNotModified):
k = await m.reply("Setting up for channel play..")
if " " in m.text:
you, me = m.text.split(" ", 1)
if me.startswith("-100"):
try:
me = int(me)
except:
await k.edit("Invalid chat id given")
await delete_messages([m, k])
return
try:
await client.get_chat_member(int(me), Config.USER_ID)
except (ValueError, PeerIdInvalid, ChannelInvalid):
LOGGER.error(
f"Given channel is private and @{Config.BOT_USERNAME} is not an admin over there.",
exc_info=True)
await k.edit(
f"Given channel is private and @{Config.BOT_USERNAME} is not an admin over there. If channel is not private , please provide username of channel."
)
await delete_messages([m, k])
return
except UserNotParticipant:
LOGGER.error(
"Given channel is private and USER account is not a member of channel."
)
await k.edit(
"Given channel is private and USER account is not a member of channel."
)
await delete_messages([m, k])
return
except Exception as e:
LOGGER.error(
f"Errors occured while getting data abount channel - {e}",
exc_info=True)
await k.edit(f"Something went wrong- {e}")
await delete_messages([m, k])
return
await k.edit(
"Searching files from channel, this may take some time, depending on number of files in the channel."
)
st, msg = await c_play(me)
if st == False:
await m.edit(msg)
else:
await k.edit(f"Succesfully added {msg} files to playlist.")
elif me.startswith("@"):
me = me.replace("@", "")
try:
chat = await client.get_chat(me)
except Exception as e:
LOGGER.error(
f"Errors occured while fetching info about channel - {e}",
exc_info=True)
await k.edit(
f"Errors occured while getting data about channel - {e}"
)
await delete_messages([m, k])
return
await k.edit(
"Searching files from channel, this may take some time, depending on number of files in the channel."
)
st, msg = await c_play(me)
if st == False:
await k.edit(msg)
await delete_messages([m, k])
else:
await k.edit(
f"Succesfully Added {msg} files from {chat.title} to playlist"
)
await delete_messages([m, k])
else:
await k.edit(
"The given channel is invalid. For private channels it should start with -100 and for public channels it should start with @\nExamples - `/cplay @VCPlayerFiles or /cplay -100125369865\n\nFor private channel, both bot and the USER account should be members of channel."
)
await delete_messages([m, k])
else:
await k.edit(
"You didn't gave me any channel. Give me a channel id or username from which i should play files . \nFor private channels it should start with -100 and for public channels it should start with @\nExamples - `/cplay @VCPlayerFiles or /cplay -100125369865\n\nFor private channel, both bot and the USER account should be members of channel."
)
await delete_messages([m, k])
def _request(self, method, arguments=None, ids=None, require_ids=False, timeout=None):
"""
Send json-rpc request to Transmission using http POST
"""
if not isinstance(method, string_types):
raise ValueError('request takes method as string')
if arguments is None:
arguments = {}
if not isinstance(arguments, dict):
raise ValueError('request takes arguments as dict')
ids = parse_torrent_ids(ids)
if len(ids) > 0:
arguments['ids'] = ids
elif require_ids:
raise ValueError('request require ids')
query = json.dumps({'tag': self._sequence, 'method': method
, 'arguments': arguments})
self._sequence += 1
start = time.time()
http_data = self._http_query(query, timeout)
elapsed = time.time() - start
LOGGER.info('http request took %.3f s' % (elapsed))
try:
data = json.loads(http_data)
except ValueError as error:
LOGGER.error('Error: ' + str(error))
LOGGER.error('Request: \"%s\"' % (query))
LOGGER.error('HTTP data: \"%s\"' % (http_data))
raise
LOGGER.debug(json.dumps(data, indent=2))
if 'result' in data:
if data['result'] != 'success':
raise TransmissionError('Query failed with result \"%s\".' % (data['result']))
else:
raise TransmissionError('Query failed without result.')
results = {}
if method == 'torrent-get':
for item in data['arguments']['torrents']:
results[item['id']] = Torrent(self, item)
if self.protocol_version == 2 and 'peers' not in item:
self.protocol_version = 1
elif method == 'torrent-add':
item = None
if 'torrent-added' in data['arguments']:
item = data['arguments']['torrent-added']
elif 'torrent-duplicate' in data['arguments']:
item = data['arguments']['torrent-duplicate']
if item:
results[item['id']] = Torrent(self, item)
else:
raise TransmissionError('Invalid torrent-add response.')
elif method == 'session-get':
self._update_session(data['arguments'])
elif method == 'session-stats':
# older versions of T has the return data in "session-stats"
if 'session-stats' in data['arguments']:
self._update_session(data['arguments']['session-stats'])
else:
self._update_session(data['arguments'])
elif method in ('port-test', 'blocklist-update', 'free-space', 'torrent-rename-path'):
results = data['arguments']
else:
return None
return results
def rank(self, sentences, theta=0.5):
LOGGER.debug('build sentences similarity matrix')
n_sentences = len(sentences)
graph = np.zeros((n_sentences, n_sentences)) # adj-matrix
for i in xrange(n_sentences):
for j in xrange(i+1, n_sentences):
weight = self.sim_word_embedding(sentences[i][1], sentences[j][1])
if weight >= theta:
graph[i, j] = weight
graph[j, i] = weight
nx_graph = nx.from_numpy_matrix(graph)
D = nx_graph
if not D.is_directed():
D = D.to_directed()
W = self._right_stochastic_graph(D)
N = W.number_of_nodes()
# power iteration
# x = (1-d) + d * x' * w
x = dict.fromkeys(W, 1.0 / N)
if self._using_matrix:
x = x.values()
w = np.zeros((N, N))
for (u, v, _w) in W.out_edges(data=True):
w[u][v] = _w['weight']
for i in xrange(self._max_iter):
x_last = x
x = 1 - self._alpha + self._alpha * np.matmul(x_last, w)
delta = x - x_last
err = np.linalg.norm(delta)
LOGGER.error('iter: %d, err: %.5f' % (i, err))
if err < N * self._tol:
return sorted(
[(x[n], sentences[n][0]) for n in xrange(len(x))], key=lambda v: v[0], reverse=True
)
else:
for i in xrange(self._max_iter):
x_last = x
x = dict.fromkeys(x_last.keys(), 0)
for n in x:
sum_in_nbr = sum([w['weight'] * x_last.get(u, 0.0) for (u, _, w) in W.in_edges(n, data=True)])
x[n] = 1 - self._alpha + self._alpha * sum_in_nbr
# check convergence
err = sum([abs(x[n] - x_last[n]) for n in x])
LOGGER.error('iter: %d, err: %.5f' % (i, err))
if err < N * self._tol:
return sorted(
[(r, sentences[n][0]) for n, r in x.items()], key=lambda v: v[0], reverse=True
)
raise nx.NetworkXError('text-rank: power iteration failed to converge in %d iterations', self._max_iter)
async def main():
await bot.start()
Config.BOT_USERNAME = (await bot.get_me()).username
LOGGER.info(f"{Config.BOT_USERNAME} Started.")
if Config.DATABASE_URI:
try:
if await db.is_saved("RESTART"):
msg = await db.get_config("RESTART")
if msg:
try:
k = await bot.edit_message_text(
msg['chat_id'],
msg['msg_id'],
text="Succesfully restarted.")
await db.del_config("RESTART")
except:
pass
await check_changes()
await sync_from_db()
except Exception as e:
LOGGER.error(
f"Errors occured while setting up database for VCPlayerBot, check the value of DATABASE_URI. Full error - {str(e)}",
exc_info=True)
Config.STARTUP_ERROR = "Errors occured while setting up database for VCPlayerBot, check the value of DATABASE_URI. Full error - {str(e)}"
LOGGER.info(
"Activating debug mode, you can reconfigure your bot with /env command."
)
await bot.stop()
from utils import debug
await debug.start()
await idle()
return
if Config.DEBUG:
LOGGER.info("Debugging enabled by user, Now in debug mode.")
Config.STARTUP_ERROR = "Debugging enabled by user, Now in debug mode."
from utils import debug
await bot.stop()
await debug.start()
await idle()
return
try:
await group_call.start()
Config.USER_ID = (await USER.get_me()).id
k = await startup_check()
if k == False:
LOGGER.error("Startup checks not passed , bot is quiting")
await bot.stop()
LOGGER.info(
"Activating debug mode, you can reconfigure your bot with /env command."
)
from utils import debug
await debug.start()
await idle()
return
if Config.IS_LOOP:
if Config.playlist:
await play()
LOGGER.info(
"Loop play enabled and playlist is not empty, resuming playlist."
)
else:
LOGGER.info(
"Loop play enabled , starting playing startup stream.")
await start_stream()
except Exception as e:
LOGGER.error(f"Startup was unsuccesfull, Errors - {e}", exc_info=True)
LOGGER.info(
"Activating debug mode, you can reconfigure your bot with /env command."
)
Config.STARTUP_ERROR = f"Startup was unsuccesfull, Errors - {e}"
from utils import debug
await bot.stop()
await debug.start()
await idle()
return
await idle()
await bot.stop()
