def convert_text_to_binary():
"""convert text data to binary
input data format:
each line looks like:
article=<d> <p> <s> word1 word2 ... </s> <s> ... </s> </p> ... </d>\tabstract=<d> <p> <s> ... </s> </p> ... </d>
"""
text_data_path = FLAGS.in_file
binary_data_path = FLAGS.out_file
assert text_data_path and binary_data_path, 'filename of text data or binary data should be provided'
if not gfile.Exists(binary_data_path):
LOGGER.debug('convert text to binary format: %s => %s', text_data_path, binary_data_path)
reader = open(text_data_path, mode='rb')
writer = open(binary_data_path, mode='wb')
for line in reader:
tf_example = example_pb2.Example()
for feature in line.strip().split(FLAGS.feature_separator):
(k, v) = feature.split('=')
tf_example.features.feature[k].bytes_list.value.extend([v])
tf_example_str = tf_example.SerializeToString()
str_len = len(tf_example_str)
writer.write(struct.pack('q', str_len))
writer.write(struct.pack('%ds' % str_len, tf_example_str))
writer.close()
reader.close()
else:
LOGGER.error('binary data exist: %s', binary_data_path)
def load_config(output_path):
config_file = os.path.join(output_path, 'config.ini')
config = configparser.ConfigParser()
if os.path.exists(config_file):
config.read(config_file)
else:
config.read_dict({
'DB': {
'host': 'localhost',
'user': '******',
'passwd': '1234',
'db': 'first_bot'
},
'Telegram': {
'API_TOKEN': 'API_Bot'
}
})
try:
with open(config_file, 'w', encoding='utf-8') as f:
config.write(f)
LOGGER.info(f'Конфигурация записана в {config_file}')
except Exception as e:
LOGGER.error(
f'Возникла ошибка при Записи данных: {e} в файл {config_file}')
LOGGER.debug(traceback.format_exc())
return config, config_file
def _http_query(self, query, timeout=None):
"""
Query Transmission through HTTP.
"""
headers = {'x-transmission-session-id': str(self.session_id)}
result = {}
request_count = 0
if timeout is None:
timeout = self._query_timeout
while True:
LOGGER.debug(
json.dumps({'url': self.url, 'headers': headers, 'query': query, 'timeout': timeout}, indent=2))
try:
result = self.http_handler.request(self.url, query, headers, timeout)
break
except HTTPHandlerError as error:
if error.code == 409:
LOGGER.info('Server responded with 409, trying to set session-id.')
if request_count > 1:
raise TransmissionError('piSession ID negotiation failed.', error)
session_id = None
for key in list(error.headers.keys()):
if key.lower() == 'x-transmission-session-id':
session_id = error.headers[key]
self.session_id = session_id
headers = {'x-transmission-session-id': str(self.session_id)}
if session_id is None:
debug_httperror(error)
raise TransmissionError('Unknown conflict.', error)
else:
debug_httperror(error)
raise TransmissionError('Request failed.', error)
request_count += 1
return result
def __init__(self, vocab_file, max_size):
self._word_to_id = {}
self._id_to_word = {}
self._count = 0
counter = 0
with open(vocab_file, 'rb') as vocab_f:
for line in vocab_f:
counter += 1
if counter % 1000 == 0:
LOGGER.debug("processing line %d", counter)
pieces = line.split()
if pieces[0] in self._word_to_id:
raise ValueError('duplicated word: %s' % pieces[0])
if pieces[0] and pieces[0].strip():
pieces[0] = pieces[0].strip()
self._word_to_id[pieces[0]] = self._count
self._id_to_word[self._count] = pieces[0]
self._count += 1
else:
sys.stderr.write('bad line: %s\n' % line)
if self._count > max_size:
raise ValueError('too many words: >%d' % max_size)
assert self.check_vocab(PAD_TOKEN) > 0
assert self.check_vocab(UNKNOWN_TOKEN) >= 0
assert self.check_vocab(SENTENCE_START) > 0
assert self.check_vocab(SENTENCE_END) > 0
def _http_query(self, query, timeout=None):
"""
Query Transmission through HTTP.
"""
headers = {'x-transmission-session-id': str(self.session_id)}
result = {}
request_count = 0
if timeout is None:
timeout = self._query_timeout
while True:
LOGGER.debug(json.dumps({'url': self.url, 'headers': headers, 'query': query, 'timeout': timeout}, indent=2))
try:
result = self.http_handler.request(self.url, query, headers, timeout)
break
except HTTPHandlerError as error:
if error.code == 409:
LOGGER.info('Server responded with 409, trying to set session-id.')
if request_count > 1:
raise TransmissionError('Session ID negotiation failed.', error)
session_id = None
for key in list(error.headers.keys()):
if key.lower() == 'x-transmission-session-id':
session_id = error.headers[key]
self.session_id = session_id
headers = {'x-transmission-session-id': str(self.session_id)}
if session_id is None:
debug_httperror(error)
raise TransmissionError('Unknown conflict.', error)
else:
debug_httperror(error)
raise TransmissionError('Request failed.', error)
request_count += 1
return result
def on_token_expired(self, jdata):
"""Handle on_token_expired json packet."""
LOGGER.debug('on_token_expired %s', str(jdata))
if 'Payload' not in jdata:
return
data = jdata['Payload']
self.client_token = data['token']
self.update_notify_state(data)
def on_req_proxy(self, jdata):
"""Handle on_req_proxy json packet."""
LOGGER.debug("on_req_proxy, %s, %s, %s, %s", self.host, self.port,
self.tunnel['lhost'], self.tunnel['lport'])
remotesession = RemoteSession(self.client_id, self.host, self.port,
self.tunnel['lhost'],
self.tunnel['lport'],
MOLO_CLIENT_APP.async_map)
remotesession.sock_connect()
def handle_close(self):
"""When closed, this method will be call. clean itself."""
self.clear()
LOGGER.debug("local session closed(%d)", id(self))
MOLO_CLIENT_APP.remote_session_dict.pop(id(self), None)
remote_session = MOLO_CLIENT_APP.remote_session_dict.get(id(self))
if remote_session:
remote_session.handle_close()
self.close()
def handle_connect(self):
"""When connected, this method will be call."""
LOGGER.debug("server connected")
self.append_connect = False
domain = MOLO_CONFIGS.get_config_object().get('domain', '')
self.send_dict_pack(
MoloSocketHelper.molo_auth(CLIENT_VERSION,
MOLO_CLIENT_APP.hass_context, '1.0.0',
domain), )
def handle_close(self):
"""When closed, this method will be call. clean itself."""
LOGGER.debug("server closed(%d)", id(self))
self.clear()
MOLO_CLIENT_APP.local_session_dict.pop(id(self), None)
local_session = MOLO_CLIENT_APP.local_session_dict.get(id(self))
if local_session:
local_session.handle_close()
self.close()
def handle_close(self):
"""When closed, this method will be call. clean itself."""
LOGGER.debug("server closed")
self.clear()
data = {}
self.update_notify_state(data, STAGE_SERVER_UNCONNECTED)
self.close()
# close all and restart
asyncore.close_all()
def on_auth_resp(self, jdata):
"""Handle on_auth_resp json packet."""
LOGGER.debug('on_auth_resp %s', str(jdata))
self.client_id = jdata['Payload']['ClientId']
self.send_dict_pack(
MoloSocketHelper.req_tunnel(self.tunnel['protocol'],
self.tunnel['hostname'],
self.tunnel['subdomain'],
self.tunnel['rport'], self.client_id))
def on_bind_status(self, jdata):
"""Handle on_bind_status json packet."""
LOGGER.debug("on_bind_status %s", str(jdata))
jpayload = jdata['Payload']
self.client_status = jpayload['Status']
jpayload['token'] = self.client_token
if self.client_status == CLIENT_STATUS_BINDED:
self.update_notify_state(jpayload, STAGE_AUTH_BINDED)
elif self.client_status == CLIENT_STATUS_UNBINDED:
self.update_notify_state(jpayload, STAGE_SERVER_CONNECTED)
def process_new_tunnel(self, jdata):
"""Handle new tunnel."""
jpayload = jdata['Payload']
self.client_id = jpayload['clientid']
self.client_token = jpayload['token']
LOGGER.debug("Get client id:%s token:%s", self.client_id,
self.client_token)
data = {}
data['clientid'] = self.client_id
data['token'] = self.client_token
self.update_notify_state(data, STAGE_SERVER_CONNECTED)
def process_tranparency_pack(self):
"""Handle transparency packet."""
localsession = MOLO_CLIENT_APP.local_session_dict.get(id(self))
if not localsession:
LOGGER.debug(
"process_tranparency_pack() localsession session not found")
self.handle_close()
return
if self.append_recv_buffer:
localsession.send_raw_pack(self.append_recv_buffer)
self.append_recv_buffer = bytes()
def on_start_proxy(self, jdata):
"""Handle Start Proxy."""
LOGGER.debug("on_start_proxy %s", str(jdata))
localsession = LocalSession(self.lhost, self.lport,
MOLO_CLIENT_APP.async_map)
MOLO_CLIENT_APP.local_session_dict[id(self)] = localsession
MOLO_CLIENT_APP.remote_session_dict[id(localsession)] = self
LOGGER.debug("remote local (%d)<->(%d)", id(self), id(localsession))
localsession.sock_connect()
self.tranparency = True
self.process_tranparency_pack()
def handle_read(self):
"""Handle read message."""
buff = self.recv(BUFFER_SIZE)
if not buff:
return
remotesession = MOLO_CLIENT_APP.remote_session_dict.get(id(self))
if not remotesession:
LOGGER.error("LocalSession handle_read remove session not found")
self.handle_close()
return
LOGGER.debug("local session handle_read %s", buff)
remotesession.send_raw_pack(buff)
def on_reset_clientid(self, jdata):
"""Handle on_reset_clientid json packet."""
local_seed = get_rand_char(32).lower()
config_file_name = MOLO_CONFIGS.get_config_object().get('domain', '')
#keep Compatibility with old version
if config_file_name and config_file_name != 'molohub':
config_file_name = CONFIG_FILE_NAME + '_' + config_file_name + '.yaml'
else:
config_file_name = CONFIG_FILE_NAME + '.yaml'
save_local_seed((config_file_name), local_seed)
LOGGER.debug("reset clientid %s to %s", self.client_id, local_seed)
self.handle_close()
def process_molo_tcp_pack(self):
"""Handle TCP packet."""
ret = self.molo_tcp_pack.recv_buffer(self.append_recv_buffer)
if ret and self.molo_tcp_pack.error_code == MoloTcpPack.ERR_OK:
self.append_recv_buffer = self.molo_tcp_pack.tmp_buffer
LOGGER.debug("RemoteSession process_molo_tcp_pack body:%s",
str(self.molo_tcp_pack.body_jdata))
self.process_json_pack(self.molo_tcp_pack.body_jdata)
if not self.tranparency:
if self.molo_tcp_pack.error_code == MoloTcpPack.ERR_MALFORMED:
LOGGER.error("tcp pack malformed!")
self.handle_close()
def parse(self, schema):
# looks at all the types called for
# matches simple types to type_methods
# stubs external calls to parent for linked types
name = schema.get('name')
instructions = []
fields = schema.get('fields', [])
for field in fields:
instructions.append(self._comprehend_field(field))
self.instructions[name] = instructions
for i in self.instructions[name]:
LOGGER.debug('Add instruction to %s : %s' % (name, i))
def on_new_tunnel(self, jdata):
"""Handle on_new_tunnel json packet."""
LOGGER.debug("on_new_tunnel %s", str(jdata))
data = jdata['OnlineConfig']
if 'ping_interval' in jdata['OnlineConfig']:
MOLO_CLIENT_APP.ping_interval = jdata['OnlineConfig'][
'ping_interval']
self.update_notify_state(data)
if jdata['Payload']['Error'] != '':
LOGGER.error('Server failed to allocate tunnel: %s',
jdata['Payload']['Error'])
return
self.client_token = jdata['Payload']['token']
self.on_bind_status(jdata)
def _load_articles(fname):
articles = []
with open(fname, 'rb') as fp:
counter = 0
for line in fp:
if counter % 1000 == 0:
LOGGER.debug('loading article: %d' % counter)
counter += 1
line = line.strip().split('\001')
doc_id = line[0]
doc_url = line[1]
sen_id = line[3]
sen = line[4]
words = [w.strip() for w in line[5].split("(")[-1].split(")")[0].split(",")]
articles.append([doc_id, doc_url, sen_id, sen, words])
articles = sorted(articles, key=lambda a: a[0])
# articles:
# {
# doc_id: {
# url: ''.
# sen: {
# sen_id: sen
# },
# words: [
# (sen_id, words)
# ]
# }
# }
grouped_articles = {}
for (doc_id, doc_url), group in groupby(articles, lambda a: (a[0], a[1])):
sen = {g[2]: (g[3], g[4]) for g in group}
words = [(sen_id, _words) for sen_id, (_, _words) in sen.items()]
cur_article = {
'url': doc_url,
'sen': sen,
'words': words
}
grouped_articles[doc_id] = cur_article
return grouped_articles
def generate_str_auth_binded(self):
"""Handle hass event: on_stage_auth_binded."""
token = quote(self.cur_data.get('token'))
opentype = self.cur_data.get('opentype')
openid = self.cur_data.get('openid')
uname = self.cur_data.get('uname')
uname = html.escape(uname)
upicture = self.cur_data.get('upicture')
LOGGER.debug("Update nofiy str opentype: %s, openid: %s, token: %s",
opentype, openid, token)
if opentype not in self.cur_data.get('platform_icon'):
opentype = 'default'
self.cur_notify_str = (self.cur_data.get('cnn_templ') %
(self.cur_data.get('platform_icon')[opentype] %
(self.molo_server_host_str), upicture, uname,
self.molo_server_host_str, token))
def sim_semantic_jaccard(self, s1, s2, alpha=0.6):
assert isinstance(s1, list) and isinstance(s2, list), 'list is required for sentences'
assert self._word2vec, 'word vec is required'
n1 = len(s1)
n2 = len(s2)
if n1 < n2:
s2, s1 = s1, s2
n2, n1 = n1, n2
LOGGER.debug('build word similar matrix')
m = np.zeros((n1, n2))
for i in xrange(n1):
for j in xrange(n2):
vs1i = np.array(self._word2vec.get(s1[i], self._unknown_vec))
vs2j = np.array(self._word2vec.get(s2[j], self._unknown_vec))
m[i, j] = self.cosine(vs1i, vs2j)
LOGGER.debug('calculate similarity')
numerator = 0.0
while True and m.size > 0:
max_m_i, max_m_j = np.unravel_index(m.argmax(), m.shape)
max_m = m[max_m_i, max_m_j]
if max_m < alpha:
break
numerator += max_m
n_row, n_col = m.shape
row = np.reshape(range(0, max_m_i) + range(max_m_i + 1, n_row), (-1, 1))
col = range(0, max_m_j) + range(max_m_j + 1, n_col)
if len(row) > 0 and len(col) > 0:
m = m[row, col]
else:
m = np.array([[]])
beta = m.size
m_diff = (1 - m).sum()
denominator = numerator + beta * m_diff
if denominator > 0:
return numerator / denominator
else:
return 1e-6
def load_settings():
try:
config, config_file = load_config(EXE_PATH)
dict_setting['host'] = config.get('DB', 'host')
dict_setting['user'] = config.get('DB', 'user')
dict_setting['passwd'] = config.get('DB', 'passwd')
dict_setting['db'] = config.get('DB', 'db')
if config.get('Telegram', 'API_TOKEN') == 'API_Bot':
LOGGER.info(f'Замените данные в файле {config_file} на реальные!')
return False
else:
dict_setting['api_token'] = config.get('Telegram', 'API_TOKEN')
except Exception as e:
LOGGER.error(f'Возникла ошибка при Загрузки данных: {e}')
LOGGER.debug(traceback.format_exc())
return dict_setting
def _load_word2vec(fname, dim=256):
length = dim + 1
word2vec = {}
with open(fname, 'rb') as fp:
counter = 0
for line in fp:
if counter % 1000 == 0:
LOGGER.debug('loading vector: %d' % counter)
line = line.strip().split()
if len(line) == length:
word2vec[line[0]] = [float(v) for v in line[1:]]
counter += 1
return word2vec
def build_vocab():
"""build vocab from raw data in text format.
input data format:
each line looks like:
article=<d> <p> <s> word1 word2 ... </s> <s> ... </s> </p> ... </d>\tabstract=<d> <p> <s> ... </s> </p> ... </d>
"""
data_path = FLAGS.in_file
vocab_path = FLAGS.out_file
assert data_path and vocab_path, 'filename of data and vocabulary should be provided'
if not gfile.Exists(vocab_path):
LOGGER.debug('build vocabulary from %s, storing it into %s', data_path, vocab_path)
vocab = {}
counter = 0
reader = codecs.open(data_path, mode='rb', encoding='utf-8')
for line in reader:
counter += 1
if counter % 1000 == 0:
LOGGER.debug("processing line %d", counter)
for feature in line.strip().split(FLAGS.feature_separator):
(k, v) = feature.split('=')
word_freq = {k: len(list(g)) for k, g in groupby(sorted(v.split())) if k not in SPECIAL_TOKENS}
for word, freq in word_freq.items():
vocab[word] = vocab.get(word, 0) + freq
reader.close()
vocab = sorted(vocab.iteritems(), key=lambda kv: kv[1], reverse=True)
vocab = [(k, v) for k, v in SPECIAL_TOKENS_FREQ.items()] + vocab
with gfile.GFile(vocab_path, mode='wb') as vocab_file:
for word, freq in vocab:
vocab_file.write(word + b'\t' + str(freq) + b'\n')
else:
LOGGER.error('vocabulary file exist: %s', vocab_path)
def debug_httperror(error):
"""
Log the Transmission RPC HTTP error.
"""
try:
data = json.loads(error.data)
except ValueError:
data = error.data
LOGGER.debug(
json.dumps(
{
'response': {
'url': error.url,
'code': error.code,
'msg': error.message,
'headers': error.headers,
'data': data,
}
},
indent=2))
def register(self, name, payload=None):
# register an entity of type 'name'
# if no payload is passed, an appropriate one will be created
count = self.type_count.get(name, 0)
count += 1
self.type_count[name] = count
if not payload:
payload = self.types[name].get()
# type_name = self.alias.get(name)
type_id = self.schema_id.get(name)
ps_id = self.schema_decorator.get(type_id)
data = self.payload_to_data(ps_id, payload)
try:
self.client.entities.create(data=data)
LOGGER.debug('Created instance # %s of type %s' % (self.type_count[name], name))
except AetherAPIException as err:
LOGGER.error('in creation of entity of type %s: %s' % (name, err))
return data
def convert_binary_to_text():
"""convert binary data to text
output data format:
each line looks like:
article=<d> <p> <s> word1 word2 ... </s> <s> ... </s> </p> ... </d>\tabstract=<d> <p> <s> ... </s> </p> ... </d>
"""
binary_data_path = FLAGS.in_file
text_data_path = FLAGS.out_file
assert binary_data_path and text_data_path, 'filename of binary data or text data should be provided'
if not gfile.Exists(text_data_path):
LOGGER.debug('convert binary to text format: %s => %s', binary_data_path, text_data_path)
reader = open(binary_data_path, mode='rb')
writer = codecs.open(text_data_path, mode='wb', encoding='utf-8')
while True:
len_bytes = reader.read(8)
if not len_bytes:
LOGGER.debug('done reading')
break
str_len = struct.unpack('q', len_bytes)[0]
tf_example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
tf_example = example_pb2.Example.FromString(tf_example_str)
examples = []
for key in tf_example.features.feature:
value = tf_example.features.feature[key].bytes_list.value[0]
value = value.decode('utf-8')
examples.append('%s=%s' % (key, value))
writer.write('%s\n' % FLAGS.feature_separator.join(examples))
writer.close()
reader.close()
else:
LOGGER.error('text data exist: %s', text_data_path)
def ping_server(self):
"""Send ping to server every ping_interval."""
while not self.is_exited:
try:
if self.molo_client:
self.set_ping_buffer(self.molo_client.ping_server_buffer())
time.sleep(self.ping_interval)
time_interval = time.time() - self.last_activate_time
LOGGER.debug("data interval: %f", time_interval)
if time_interval > PROXY_TCP_CONNECTION_ACTIVATE_TIME:
LOGGER.info("connection timeout, reconnecting server")
self.molo_client.handle_close()
self.reset_activate_time()
except Exception as exc:
print("ping_server(): " + str(exc))
asyncore.close_all()
self.molo_client.sock_connect()
time.sleep(RECONNECT_INTERVAL)
LOGGER.info("moloserver reconnecting...")
def load(self):
# loads schemas and project schemas from aether client
LOGGER.debug('Loading schemas from Aether Kernel')
for schema in self.client.schemas.paginated('list'):
name = schema.name
LOGGER.debug('Loading schema for type %s \n%s' % (name, schema))
_id = schema.id
definition = schema.definition
if isinstance(definition, str):
definition = json.loads(definition)
if isinstance(definition, list):
full_name = [
obj.get('name')
for obj in definition
if obj.get('name').endswith(name)
][0]
else:
full_name = definition.get('name')
namespace = definition.get('namespace')
if namespace and not name in namespace:
full_name = namespace + '.' + name
self.types[full_name] = DataMocker(full_name, json.dumps(definition), self)
self.names[name] = full_name
self.names[full_name] = name
self.types[name] = self.types[full_name]
self.alias[full_name] = name
self.alias[name] = full_name
self.schema_id[name] = _id
self.schema_id[full_name] = _id
self.schema_id[_id] = name
for ps in self.client.schemadecorators.paginated('list'):
schema_id = ps.schema
_id = ps.id
self.schema_decorator[schema_id] = _id
self.schema_decorator[_id] = schema_id
def debug_httperror(error):
"""
Log the Transmission RPC HTTP error.
"""
try:
data = json.loads(error.data)
except ValueError:
data = error.data
LOGGER.debug(
json.dumps(
{
'response': {
'url': error.url,
'code': error.code,
'msg': error.message,
'headers': error.headers,
'data': data,
}
},
indent=2
)
)
def proxy_loop(self):
"""Handle main loop and reconnection."""
self.molo_client.sock_connect()
while not self.is_exited:
try:
asyncore.loop(map=self.async_map)
except asyncore.ExitNow as exc:
logging.exception(exc)
LOGGER.error("asyncore.loop exception")
if not self.is_exited:
try:
asyncore.close_all()
self.molo_client.sock_connect()
time.sleep(RECONNECT_INTERVAL)
LOGGER.info("moloserver reconnecting...")
except Exception as exc:
print("proxy_loop(): " + str(exc))
LOGGER.info("reconnect failed, retry...")
time.sleep(RECONNECT_INTERVAL)
asyncore.close_all()
LOGGER.debug("proxy exited")
def update_state(self, data):
"""Update UI state."""
last_stage = self.cur_data.get('stage')
last_notify_str = self.get_notify_str()
# Update data
self.cur_data.update(data)
cur_stage = self.cur_data.get('stage')
LOGGER.debug("cur_data %s", str(self.cur_data))
# Generate notify string according to stage
if cur_stage in self.generate_str_func_bind_map:
self.generate_str_func_bind_map[cur_stage]()
# If notify string changed, inform UI to update
if last_notify_str == self.get_notify_str():
return
# If stage changed, log new stage
if cur_stage != last_stage:
LOGGER.info(self.state_log_str[cur_stage])
# Inform UI to update
fire_molohub_event(self.molo_server_host_str, data)
def register():
project = register_project()
project_id = project.id
if not project_id:
LOGGER.error('project could not be registerd, does it already exist?')
sys.exit(0)
schema_info = schema()
if not schema_info:
raise ValueError('No schemas registered')
LOGGER.debug(schema_info)
schema_ids = {obj.name: obj.id for obj in schema_info}
LOGGER.debug(schema_ids)
schema_decorators = schema_decorator(project_id, schema_ids)
LOGGER.debug(schema_decorators)
ps_ids = {ps.name: ps.id for ps in schema_decorators.values()}
ms = mappingset(project_id)
sub_id = mapping(project_id, ms.id, ps_ids)
LOGGER.debug(sub_id)
def send_raw_pack(self, raw_data):
"""Write raw data pack to write buffer."""
self.append_send_buffer += raw_data
LOGGER.debug("local session send_raw_pack %s", raw_data)
if not self.append_connect:
self.handle_write()
def rank(self, sentences, theta=0.5):
LOGGER.debug('build sentences similarity matrix')
n_sentences = len(sentences)
graph = np.zeros((n_sentences, n_sentences)) # adj-matrix
for i in xrange(n_sentences):
for j in xrange(i+1, n_sentences):
weight = self.sim_word_embedding(sentences[i][1], sentences[j][1])
if weight >= theta:
graph[i, j] = weight
graph[j, i] = weight
nx_graph = nx.from_numpy_matrix(graph)
D = nx_graph
if not D.is_directed():
D = D.to_directed()
W = self._right_stochastic_graph(D)
N = W.number_of_nodes()
# power iteration
# x = (1-d) + d * x' * w
x = dict.fromkeys(W, 1.0 / N)
if self._using_matrix:
x = x.values()
w = np.zeros((N, N))
for (u, v, _w) in W.out_edges(data=True):
w[u][v] = _w['weight']
for i in xrange(self._max_iter):
x_last = x
x = 1 - self._alpha + self._alpha * np.matmul(x_last, w)
delta = x - x_last
err = np.linalg.norm(delta)
LOGGER.error('iter: %d, err: %.5f' % (i, err))
if err < N * self._tol:
return sorted(
[(x[n], sentences[n][0]) for n in xrange(len(x))], key=lambda v: v[0], reverse=True
)
else:
for i in xrange(self._max_iter):
x_last = x
x = dict.fromkeys(x_last.keys(), 0)
for n in x:
sum_in_nbr = sum([w['weight'] * x_last.get(u, 0.0) for (u, _, w) in W.in_edges(n, data=True)])
x[n] = 1 - self._alpha + self._alpha * sum_in_nbr
# check convergence
err = sum([abs(x[n] - x_last[n]) for n in x])
LOGGER.error('iter: %d, err: %.5f' % (i, err))
if err < N * self._tol:
return sorted(
[(r, sentences[n][0]) for n, r in x.items()], key=lambda v: v[0], reverse=True
)
raise nx.NetworkXError('text-rank: power iteration failed to converge in %d iterations', self._max_iter)
def handle_connect(self):
"""When connected, this method will be call."""
LOGGER.debug("local session connected(%d)", id(self))
self.append_connect = False
def stop_reverse_proxy(self):
"""Stop application, close all sessions."""
LOGGER.debug("stopping reverse proxy")
self.is_exited = True
asyncore.close_all()
def _request(self, method, arguments=None, ids=None, require_ids=False, timeout=None):
"""
Send json-rpc request to Transmission using http POST
"""
if not isinstance(method, string_types):
raise ValueError('request takes method as string')
if arguments is None:
arguments = {}
if not isinstance(arguments, dict):
raise ValueError('request takes arguments as dict')
ids = parse_torrent_ids(ids)
if len(ids) > 0:
arguments['ids'] = ids
elif require_ids:
raise ValueError('request require ids')
query = json.dumps({'tag': self._sequence, 'method': method
, 'arguments': arguments})
self._sequence += 1
start = time.time()
http_data = self._http_query(query, timeout)
elapsed = time.time() - start
LOGGER.info('http request took %.3f s' % (elapsed))
try:
data = json.loads(http_data)
except ValueError as error:
LOGGER.error('Error: ' + str(error))
LOGGER.error('Request: \"%s\"' % (query))
LOGGER.error('HTTP data: \"%s\"' % (http_data))
raise
LOGGER.debug(json.dumps(data, indent=2))
if 'result' in data:
if data['result'] != 'success':
raise TransmissionError('Query failed with result \"%s\".' % (data['result']))
else:
raise TransmissionError('Query failed without result.')
results = {}
if method == 'torrent-get':
for item in data['arguments']['torrents']:
results[item['id']] = Torrent(self, item)
if self.protocol_version == 2 and 'peers' not in item:
self.protocol_version = 1
elif method == 'torrent-add':
item = None
if 'torrent-added' in data['arguments']:
item = data['arguments']['torrent-added']
elif 'torrent-duplicate' in data['arguments']:
item = data['arguments']['torrent-duplicate']
if item:
results[item['id']] = Torrent(self, item)
else:
raise TransmissionError('Invalid torrent-add response.')
elif method == 'session-get':
self._update_session(data['arguments'])
elif method == 'session-stats':
# older versions of T has the return data in "session-stats"
if 'session-stats' in data['arguments']:
self._update_session(data['arguments']['session-stats'])
else:
self._update_session(data['arguments'])
elif method in ('port-test', 'blocklist-update', 'free-space', 'torrent-rename-path'):
results = data['arguments']
else:
return None
return results
