def handle(self, *args, **options):
if options['daemonize']:
daemonize()
settings.LOG_TO_STREAM = True
r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
if options['initialize']:
feeds = Feed.objects.filter(num_subscribers__gte=1).order_by('?')
print 'Query feeds done with num of feeds',len(feeds)
r.ltrim('freeze_feeds',1,0)
pipeline = r.pipeline()
for feed in feeds:
pipeline.rpush('freeze_feeds',feed.pk)
pipeline.execute()
print 'Initialize freeze_feeds done'
feed_id = r.lpop('freeze_feeds')
while feed_id:
try:
frozen_num = MStory.freeze_feed(int(feed_id))
if frozen_num > 0:
r.rpush('freeze_feeds',feed_id)
except Exception, e:
logging.error(str(e)+\
traceback.format_exc()+'\n'+\
'Error from: freeze_feeds\n')
feed_id = r.lpop('freeze_feeds')
def current(roster, cli, config):
'''Print information on the current shift in the roster.'''
# roster.current return a *list* of all the people on duty
shifts = roster.current
if len(shifts) == 1:
[current] = shifts
elif len(shifts) == 0:
log.error('Nobody is on duty.')
now = datetime.datetime.now(tz=pytz.UTC)
current = Shift(now, now, None, None, None)
else:
log.error('Several people where on duty, picking a random one.')
for counter, shift in enumerate(shifts, 1):
log.error('On duty #{}: {}'.format(counter, shift))
current = choice(shifts)
# Replace missing fields with fallback ones
if not current.email:
current.email = config['fallback.email']
if current.name is not None:
log.error('Missing email address for "{}"'.format(current.name))
if not current.phone:
current.phone = config['fallback.phone']
if current.name is not None:
log.error('Missing phone number for "{}"'.format(current.name))
current.name = current.name or 'Fallback Contact Details'
# Compute what fields to output
fields = ('start', 'end', 'name', 'email', 'phone')
mask = []
for attr_name in fields:
mask.append(cli[attr_name])
if not any(mask):
mask = [True] * 5 # No explicit field, means all fields
bits = [val for val, flag in zip(current.as_string_tuple, mask) if flag]
print('\t'.join(bits))
def load_from_rawxls(file_path):
"""从 xlsx 文件恢复数据到 lang.csv 的格式
xlsx 文件可能是 <name, desc> 模式，也可能是 list<text> 模式。
Args:
file_path (str): xlsx 文件路径
Returns:
category (str): category from lang_def
csv_data (list[str]): list of [file_id, unknown, index, text]，不带前导0
"""
data = load_xls(file_path)[1:]
# 判断文件模式
id_split = data[0][1].split('-')
if len(id_split) > 3 and id_split[-1].isdigit() and id_split[-2].isdigit() and id_split[-3].isdigit():
# list of text
return load_from_list_category(data)
elif len(id_split) > 1 and id_split[-1].isdigit():
# name_desc
return load_from_pair_category(data)
else:
log.error('load %s failed.' % file_path)
return '', []
def _is_valid_link(self, link):
"""
Return True if given link is non document, Since this is not a perfect way to check
but it avoids a call to server.
"""
# Check ONLY_ROOTDOMAIN
scheme, netloc, path, params, query, fragment = urlparse(link)
try:
if get_tld(self.base_url) == get_tld(link) and not ONLY_ROOTDOMAIN:
# if get_tld(self.base_url) == get_tld(link):
return False
except Exception as e:
log.error(str(e), self.base_url, link)
# Need to add more
DOC_EXT = [".pdf", ".xmls", ".docx", ".odt"]
try:
urlPath = [i for i in (path.split('/')) if i]
file_name = urlPath[-1]
ext = file_name.split(".")[-1]
except IndexError:
# Its just a root URL
return True
return ext not in DOC_EXT
def isType(key, value, expected):
try:
assert isinstance(value, expected)
except AssertionError:
log.error("Key '"+key+"' is of type '"+type(value).__name__+"';",
"expected '"+expected.__name__+"'.",
"(value: "+str(value)+")")
def required(a, group=None, name=None):
k = key(a, group)
if k.required == True:
if name:
log.error("Missing '"+a+"' key in "+group+" '"+name+"'")
else:
log.error("Missing key '"+a+"' in group '"+group+"'")
def push(self):
"""Push todo to gist.github.com"""
github = Github()
gist_id = GistId().get()
token = GithubToken().get()
github.login(token)
if not self.todo.name:
name = "Todo"
else:
name = self.todo.name
files = {
name: {
"content": self.todo_content
}
}
log.info(
"Pushing '%s' to https://gist.github.com/%s .." % (name, gist_id)
)
response = github.edit_gist(gist_id, files=files)
if response.status_code == 200:
log.ok("Pushed success.")
elif response.status_code == 401:
log.warning("Github token out of date, empty the old token")
GithubToken().save('') # empty the token!
self.push() # and repush
else:
log.error("Pushed failed. %d" % response.status_code)
def get_task_by_id(self, index):
"""return task object by its index, if not found, fatal error."""
index -= 1
tasks = self.todo.tasks
if index >= len(tasks):
log.error("Task not found.")
return tasks[index]
def load_from_langxls(file_path, lang, need_check=False, load_ui=False):
"""从 xlsx 文件中读取 lang.csv 的翻译
xlsx 文件可能是 <name, desc> 模式，也可能是 list<text> 模式。
Args:
file_path (str): xlsx 文件路径
lang (str): "zh"/"en", 读中文还是英文
need_check (bool): 是否检查
load_ui (bool): 是否读取ui汉化文件的内容
Returns:
category (str): category from lang_def
translated_data (list[str]): list of [file_id, unknown, index, text]，不带前导0
"""
data = load_xls(file_path)[1:]
# 判断文件模式
id_split = data[0][1].split('-')
if len(id_split) > 3 and id_split[-1].isdigit() and id_split[-2].isdigit() and id_split[-3].isdigit():
# list of text
return load_from_list_category(data, lang, need_check)
elif len(id_split) > 1 and id_split[-1].isdigit():
# name_desc
return load_from_pair_category(data, lang, need_check)
elif len(id_split) == 1 and '_' in id_split[0] \
and id_split[0][0].isalpha() and id_split[0][-1].isalpha():
if load_ui:
return load_from_ui_fake(data, lang, need_check)
else:
log.info('skip ui file %s.' % file_path)
return '', []
else:
log.error('load %s failed.' % file_path)
return '', []
def get(self):
"""
call this method to get a token::
token = GithubToken().get()
what the get() does:
1) read from "~/.todo/token"
2) check if the token read is empty.
(yes)-> 1) if empty,
ask user for user&passwd to access api.github.com
2) fetch the token, set to this instance and store it.
3) return the token (a string)
"""
if self.is_empty:
user = ask_input.text("Github user:"******"Password for %s:" % user)
log.info("Authorize to github.com..")
response = Github().authorize(user, password)
if response.status_code == 201:
# 201 created
log.ok("Authorized success.")
# get token from dict
token = response.json()["token"]
self.save(token)
else:
log.error("Authorization failed. %d" % response.status_code)
return self.content
def fetch_request(self):
try:
proxies = {"http": settings.COW_PROXY_HANDLER, "https": settings.COW_PROXY_HANDLER}
r = requests.get(self.story.story_permalink, headers=self.headers, verify=False, proxies=proxies)
# r = requests.get(self.story.story_permalink, headers=self.headers, verify=False)
except (
AttributeError,
SocketError,
requests.ConnectionError,
requests.models.MissingSchema,
requests.sessions.InvalidSchema,
), e:
logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % e)
logging.error(
"fetch_request with:ERROR \n"
+ str(e)
+ " feed_id:"
+ str(self.story.story_feed_id)
+ " stroy_link:"
+ str(self.story.story_permalink)
)
if settings.SEND_ERROR_MAILS:
mail_admins(
"Error in text_importer fetch_request",
str(e)
+ "\n"
+ " feed_id:"
+ str(self.story.story_feed_id)
+ "\n"
+ " stroy_link:"
+ str(self.story.story_permalink)
+ "\n"
+ traceback.format_exc(),
)
return
def run(self, target=None, tid=None):
if target is None:
log.critical("Please set --target param")
sys.exit()
if tid is None:
log.critical("Please set --tid param")
sys.exit()
# Statistic Code
p = subprocess.Popen(
['cloc', target], stdout=subprocess.PIPE)
(output, err) = p.communicate()
rs = output.split("\n")
for r in rs:
r_e = r.split()
if len(r_e) > 3 and r_e[0] == 'SUM:':
t = CobraTaskInfo.query.filter_by(id=tid).first()
if t is not None:
t.code_number = r_e[4]
try:
db.session.add(t)
db.session.commit()
log.info("Statistic code number done")
except Exception as e:
log.error("Statistic code number failed" + str(e.message))
def fetch(self, skip_save=False):
# =====================
# =Modified by SongJun=
# =====================
try:
request = urllib2.Request(self.story.story_permalink, headers=self.headers)
opener = urllib2.build_opener(
urllib2.ProxyHandler({"http": settings.COW_PROXY_HANDLER, "https": settings.COW_PROXY_HANDLER})
)
# opener = urllib2.build_opener()
text = opener.open(request).read()
# logging.info(text)
# Add by Xinyan Lu: some websites always return gzip files
if text[:6] == "\x1f\x8b\x08\x00\x00\x00":
text = gzip.GzipFile(fileobj=cStringIO.StringIO(text)).read()
except httplib.IncompleteRead as e:
text = e.partial
except (Exception), e:
logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % e)
logging.error(
"error fetch_request"
+ str(e)
+
# ' feed_id:'+str(self.story.story_feed_id)+\
" stroy_link:"
+ str(self.story.story_permalink)
)
return
def on_receive(self, msg):
try:
return getattr(self, msg['func'])(msg['msg'])
except Exception, e:
log.error('Get error: %s', e, exc_info=True)
# TODO fix return
return None
def run(self, feed_pks, **kwargs):
try:
from apps.rss_feeds.models import Feed
#from apps.statistics.models import MStatistics
r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
#mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
#compute_scores = bool(mongodb_replication_lag < 10)
options = {
# 'quick': float(MStatistics.get('quick_fetch', 0)),
# 'compute_scores': compute_scores,
# 'mongodb_replication_lag': mongodb_replication_lag,
}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
feed = Feed.get_by_id(feed_pk)
if not feed or feed.pk != int(feed_pk):
logging.info(" ---> ~FRRemoving feed_id %s from tasked_feeds queue, points to %s..." % (feed_pk, feed and feed.pk))
r.zrem('tasked_feeds', feed_pk)
if feed:
feed.update(**options)
except Exception, e:
logging.error(str(e)+\
traceback.format_exc()+'\n'+\
'error from: UpdateFeeds\n')
if settings.SEND_ERROR_MAILS:
mail_admins("Error in UpdateFeeds",str(e)+'\n'+traceback.format_exc())
def create_zip(archive, files):
'''Creates a zip file containing the files being backed up.'''
import zipfile
from utils.misc import add_file_hash
try:
# zipfile always follows links
with zipfile.ZipFile(archive, 'w') as zipf:
zipf.comment = 'Created by s3-backup'
for f in files:
f = f.strip()
if os.path.exists(f):
zipf.write(f)
add_file_hash(archive, f)
log.debug('Added %s.' % f)
else:
log.error('%s does not exist.' % f)
if zipf.testzip() != None:
log.error('An error occured creating the zip archive.')
except zipfile.BadZipfile:
# I assume this only happens on reads? Just in case...
log.critical('The zip file is corrupt.')
except zipfile.LargeZipFile:
log.critical('The zip file is greater than 2 GB.'
' Enable zip64 functionality.')
def parse(self, file_name):
errors = []
patterns = {}
patterns['type'] = r'^(?P<name>\w+),\*,\*,\s*(?P<type>(?:struct |unsigned |const )?[*\w]+(?: \*| const)?),\*,\*,?$'
patterns['comment'] = r'^#.*$'
with open(file_name, 'r') as dottm_file:
for line in [ l.strip() for l in dottm_file if len(l.strip()) > 0 ]:
for k, regex in patterns.items():
match = re.match(regex, line)
if match:
if k == 'type':
type_name = match.group('type')
if type_name == '*':
type_name = match.group('name')
type_mappings[match.group('name')] = type_name
break
else:
errors.append('U: ' + line)
for e in errors:
log.error(file_name + ': ' + e)
def get_links(self, url, page_depth=0):
"""
Get the links from url
"""
if (self._is_url(url) and
page_depth <= MAX_DEPTH and
not self._is_crawled(url)):
try:
redis.hset(URLS_CRAWLLING, base64.b64encode(url.encode('UTF-8')), "")
response = requests.get(url)
if (response.status_code//100 == 2 and
self._is_html(response)):
redis.hset(URLS_CRAWLLING, base64.b64encode(url.encode('UTF-8')), response.content)
status, links = Parser(self._base_url(url)).parse_href(response.content)
log.debug("Page Depth {0} URL {2} Total count {1}".format(page_depth, len(links), url))
for link in links:
self.get_links(link, page_depth+1)
except Exception as e:
log.error(str(e))
raise
def module(name, parent=None):
if parent == None:
return importlib.import_module(name)
else:
if not type(parent) == ModuleType:
log.error("Module '"+str(parent)+"' is not a module!")
return importlib.import_module(parent.__name__+'.'+name)
def isMissingKeys(keylist):
if len(_missingkeys):
msg = "The following keys are missing from the packfile!\n"
for k in _missingkeys:
msg += "\n- %-16s %-50s (e.g. %s)" % (k,
keylist.description(k),
keylist.example(k))
log.error(msg)
def verify(self, receipt=None):
trans_receipt = receipt
trans = simplejson.loads(trans_receipt)
result = True
result_code = PurchaseResultCode.Value("UNKNOWN")
try:
resp = self.do_verify(trans_receipt)
except Exception, e:
log.error("IAP - verify exception: %s" % str(e))
def add_root(tree):
if not tree.group == None:
log.error("Not root tree:", tree.name)
for t in tree_table:
if t.name == tree.name:
log.note("Root tree is defined twice:", tree.name)
tree_table.append(tree)
def run_server():
host = settings.LISTEN_HOST
port = settings.LISTEN_PORT
#log.info('version=%s', settings.GAME_VERSION)
try:
log.info('serving on %s:%s...', host, port)
WSGIServer((host, port), app).serve_forever()
except Exception, e:
log.error('Server', exc_info=e)
def run(self, target=None, tid=None, pid=None):
if target is None:
log.critical("Please set --target param")
sys.exit()
if tid is not None:
task_id = tid
# Start Time For Task
t = CobraTaskInfo.query.filter_by(id=tid).first()
if t is None:
log.critical("Task id doesn't exists.")
sys.exit()
if t.status not in [0, 1]:
log.critical("Task Already Scan.")
sys.exit()
t.status = 1
t.time_start = int(time.time())
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
log.error("Set start time failed" + str(e.message))
else:
task_id = None
target_type = self.parse_target(target)
if target_type is False:
log.error("""
Git Repository: must .git end
SVN Repository: can http:// or https://
Directory: must be local directory
File: must be single file or tar.gz/zip/rar compress file
""")
from engine import static
s = static.Static(target, task_id=task_id, project_id=pid)
if target_type is 'directory':
s.analyse()
elif target_type is 'compress':
from utils.decompress import Decompress
# load an compressed file. only tar.gz, rar, zip supported.
dc = Decompress(target)
# decompress it. And there will create a directory named "222_test.tar".
dc.decompress()
s.analyse()
elif target_type is 'file':
s.analyse()
elif target_type is 'git':
from pickup.GitTools import Git
g = Git(target, branch='master')
g.get_repo()
if g.clone() is True:
s.analyse()
else:
log.critical("Git clone failed")
elif target_type is 'svn':
log.warning("Not Support SVN Repository")
def call_action(module, action_name, method="GET", get_p=None, post_p=None, item=None):
"""
Intenta llamar a la funcion y pasar los parametros apropiados
"""
try:
a = getattr(module, action_name)
except AttributeError, e: # diferencia entre fallo por modulo o accion
log.error("module %s doesn't have a %s function" % (module.__name__, action_name))
log.error("%s" % e)
redirect("/400")
def get_module(module_name):
"""
recupera el modulo
"""
try:
m = import_module(module_name)
log.debug("modulo %s cargado" % module_name)
except ImportError, e:
log.error("No se pudo importar %s: %s" % (module_name, e))
m = None
def parse_message(name, body):
#cls = globals()[name]
cls = getattr(protocol_pb2, name)
event = None
try:
event = cls()
msg=body
Base64=base64.b64decode(msg)
event.ParseFromString(Base64)
except Exception, e:
log.error('Error parse_message: %s', e)
def setPriority(self, idx, priority):
query = QSqlQuery(self.db)
query.prepare("update TodoList set priority = ? where id = ?")
query.addBindValue(priority)
query.addBindValue(idx)
ok = query.exec_()
query.finish()
if not ok:
log.error("Update TodoList Priority Failed: " + query.lastError().text())
return ok
def setContent(self, idx, content):
query = QSqlQuery(self.db)
query.prepare("update TodoList set content = ? where id = ?")
query.addBindValue(content)
query.addBindValue(idx)
ok = query.exec_()
query.finish()
if not ok:
log.error("Update TodoList Content Failed: " + query.lastError().text())
return ok
def test_encode_error(self):
for i, (data, expected) in enumerate(test_data_encode_error):
try:
encode(data)
except expected:
log.error(
"**IGNORE ERROR LOG** This exception was raised by a test")
except (Exception), e:
debug_print(i, data, expected, e)
raise
else:
debug_print(i, data, expected, 'NO EXCEPTION RAISED')
assert False
def fit(self):
self.alpha = -1 if self.alpha is None else self.alpha
self.beta = -1 if self.beta is None else self.beta
self.l = -1 if self.l is None else self.l
self.c = -1 if self.c is None else self.c
if self.distance == self.SIM_ASYMCOSINE and not (0 <= self.alpha <= 1):
log.error(
'Invalid parameter alpha in asymmetric cosine Similarity_MFD!')
return
if self.distance == self.SIM_TVERSKY and not (0 <= self.alpha <= 1
and 0 <= self.beta <= 1):
log.error(
'Invalid parameter alpha/beta in tversky Similarity_MFD!')
return
if self.distance == self.SIM_P3ALPHA and self.alpha is None:
log.error('Invalid parameter alpha in p3alpha Similarity_MFD')
return
if self.distance == self.SIM_RP3BETA and self.alpha is None and self.beta is None:
log.error('Invalid parameter alpha/beta in rp3beta Similarity_MFD')
return
if self.distance == self.SIM_SPLUS and not (
0 <= self.l <= 1 and 0 <= self.c <= 1 and 0 <= self.alpha <= 1
and 0 <= self.beta <= 1):
log.error(
'Invalid parameter alpha/beta/l/c in s_plus Similarity_MFD')
return
# compute and stores the Similarity_MFD matrix using one of the distance metric: S = R•R'
if self.distance == self.SIM_COSINE:
self._sim_matrix = sim.cosine(self.matrix,
k=self.k,
shrink=self.shrink,
threshold=self.threshold,
binary=self.implicit)
elif self.distance == self.SIM_ASYMCOSINE:
self._sim_matrix = sim.asymmetric_cosine(self.matrix,
k=self.k,
shrink=self.shrink,
threshold=self.threshold,
binary=self.implicit,
alpha=self.alpha)
elif self.distance == self.SIM_JACCARD:
self._sim_matrix = sim.jaccard(self.matrix,
k=self.k,
shrink=self.shrink,
threshold=self.threshold,
binary=self.implicit)
elif self.distance == self.SIM_DICE:
self._sim_matrix = sim.dice(self.matrix,
k=self.k,
shrink=self.shrink,
threshold=self.threshold,
binary=self.implicit)
elif self.distance == self.SIM_TVERSKY:
self._sim_matrix = sim.tversky(self.matrix,
k=self.k,
shrink=self.shrink,
threshold=self.threshold,
binary=self.implicit,
alpha=self.alpha,
beta=self.beta)
elif self.distance == self.SIM_P3ALPHA:
self._sim_matrix = sim.p3alpha(self.matrix,
k=self.k,
shrink=self.shrink,
threshold=self.threshold,
binary=self.implicit,
alpha=self.alpha)
elif self.distance == self.SIM_RP3BETA:
self._sim_matrix = sim.rp3beta(self.matrix,
k=self.k,
shrink=self.shrink,
threshold=self.threshold,
binary=self.implicit,
alpha=self.alpha,
beta=self.beta)
elif self.distance == self.SIM_SPLUS:
self._sim_matrix = prep.normalize(sim.s_plus(
self.matrix,
k=self.k,
shrink=self.shrink,
threshold=self.threshold,
binary=self.implicit,
l=self.l,
t1=self.alpha,
t2=self.beta,
c=self.c),
norm='l2',
axis=0)
else:
log.error('Invalid distance metric: {}'.format(self.distance))
return self._sim_matrix
def recommend(self, userid, urm=None, N=10, filter_already_liked=True, with_scores=False, items_to_exclude=[]):
if not userid >= 0:
log.error('Invalid user id')
return None
return self.recommend_batch([userid], urm, N, filter_already_liked, with_scores, items_to_exclude)
def toc(visitor, block, headers=None, relations=None):
"""
Format:
{% toc max_depth=2,class=multiple %}
file1.md
file2.md
{% endtoc %}
"""
s = []
s.append('<div class="toc">\n')
depth = int(block['kwargs'].get('max_depth', '1'))
if 'class' in block['kwargs']:
s.append('<ul class=%s>\n' % block['kwargs']['class'])
else:
s.append('<ul>\n')
prev = {'prev': {}, 'next': {}, 'current': {}}
for fname in block['body'].splitlines():
hs = headers.get(fname, [])
if not hs:
log.error("File %s can't be found, will be skipped" % fname)
continue
def make_title(x):
if x['id']:
_id = '#' + x['id']
else:
_id = ''
return '<li><a href="%s%s">%s</a></li>' % (x['link'], _id,
x['title'])
_fname, _ext = os.path.splitext(fname)
last = 1
title = _fname
for x in hs:
level = x['level']
#fetch title, so an article should be one h1 subject
if level == 1:
title = x['title']
if level > depth:
continue
if level == last: #same
s.append(make_title(x))
elif level == last - 1: #reindent
s.append('</ul>\n')
s.append(make_title(x))
last -= 1
elif level == last + 1: #indent
s.append('<ul>\n')
s.append(make_title(x))
last += 1
else:
pass
for i in range(last, 1, -1):
s.append('</ul>\n')
#process prev and next
c = {'link': _fname + '.html', 'title': title}
current = relations[_fname] = {
'prev': {},
'next': {},
'current': c,
}
p = prev['current']
if p:
current['prev'] = {'link': p['link'], 'title': p['title']}
prev['next'] = c
prev = current
s.append('</ul>\n')
s.append('</div>\n')
return ''.join(s)
async def post(cls,
url,
params=None,
body=None,
headers=None,
encode_type='utf-8',
decode_type='utf-8',
parse_json=True,
timeout=30):
""" HTTP POST 请求
@param url 请求url
@param params 请求的uri qurey参数
@param body 请求的body参数
@param headers 请求的header参数
@param encode_type 请求body编码格式，默认使用utf-8编码
@param decode_type 返回body解码格式，默认使用utf-8解码
@param parse_json 是否解析返回body为json格式，默认为True
@param timeout 请求超时时间，默认30秒
@return data 返回的http body
"""
if params:
url = url_concat(url, params)
if body:
if not encode_type:
pass
elif encode_type == 'utf-8':
body = json.dumps(body)
else:
body = urlencode(body, encoding=encode_type)
http_client = AsyncHTTPClient()
response = await http_client.fetch(url,
method='POST',
body=body,
headers=headers,
request_timeout=timeout)
if response.code not in (200, 201, 202, 203, 204, 205, 206):
logger.error('url:',
url,
'post data:',
body,
'response code:',
response.code,
'response body:',
response.body,
caller=cls)
msg = '请求url失败: {url}'.format(url=url)
raise exceptions.CustomException(msg=msg)
if response.body:
data = response.body
if decode_type:
data = data.decode(decode_type)
if parse_json:
return json.loads(data)
else:
return data
else:
return None
def exec_test(config_data):
add_test_info = AddTestInfo(11, 'salt key and salt key id')
add_test_info.started_info()
try:
api_salt_key = APISaltKeyOps(**config_data)
api_salt_key.get_salt_key()
api_salt_key.get_salt_key_id()
add_test_info.status('ok')
except Exception, e:
log.error('test error')
log.error(e)
add_test_info.status('error')
add_test_info.completed_info()
if __name__ == '__main__':
config_data = config.get_config()
if not config_data['auth']:
log.error('auth failed')
else:
exec_test(config_data)
def _test_ping_error(self):
outgoing_query = OutgoingPingQuery(tc.CLIENT_ID)
#outgoing_query.my_id = CLIENT_ID
#outgoing_query.tid = tc.TID
# TID and ARGS ID are None
assert_raises(MsgError, outgoing_query.encode)
log.error(
"**IGNORE 2 ERROR LOGS** This exception was raised by a test")
outgoing_query = OutgoingPingQuery()
outgoing_query.my_id = tc.CLIENT_ID
#outgoing_query.tid = tc.TID
assert_raises(MsgError, outgoing_query.encode)
log.error(
"**IGNORE 2 ERROR LOGS** This exception was raised by a test")
outgoing_query = OutgoingPingQuery()
#outgoing_query.my_id = tc.CLIENT_ID
outgoing_query.tid = tc.TID
assert_raises(MsgError, outgoing_query.encode)
log.error(
"**IGNORE 2 ERROR LOGS** This exception was raised by a test")
outgoing_query = OutgoingPingQuery()
assert_raises(MsgError, outgoing_query.__setattr__, 'my_id', '')
log.error(
"**IGNORE 2 ERROR LOGS** This exception was raised by a test")
outgoing_query = OutgoingPingQuery()
outgoing_query.my_id = tc.CLIENT_ID
outgoing_query.tid = 567
data = outgoing_query.encode()
assert_raises(MsgError, decode, data)
log.error(
"**IGNORE 2 ERROR LOGS** This exception was raised by a test")
outgoing_query = OutgoingPingQuery()
outgoing_query.my_id = tc.CLIENT_ID
outgoing_query.tid = tc.TID
data = outgoing_query.encode()
data += 'this string ruins the bencoded msg'
assert_raises(MsgError, decode, data)
log.error(
"**IGNORE 2 ERROR LOGS** This exception was raised by a test")
outgoing_response = OutgoingPingResponse(tc.TID, tc.SERVER_ID)
outgoing_response.tid = None
assert_raises(MsgError, outgoing_response.encode)
log.error(
"**IGNORE ERROR LOGS** This exception was raised by a test")
def run(self, **kwargs):
try:
from apps.rss_feeds.models import Feed
settings.LOG_TO_STREAM = True
now = datetime.datetime.utcnow()
start = time.time()
r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
tasked_feeds_size = r.zcard('tasked_feeds')
hour_ago = now - datetime.timedelta(hours=1)
r.zremrangebyscore('fetched_feeds_last_hour', 0, int(hour_ago.strftime('%s')))
now_timestamp = int(now.strftime("%s"))
queued_feeds = r.zrangebyscore('scheduled_updates', 0, now_timestamp)
print len(queued_feeds)
r.zremrangebyscore('scheduled_updates', 0, now_timestamp)
logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
len(queued_feeds),
r.zcard('tasked_feeds'),
r.scard('queued_feeds'),
r.zcard('scheduled_updates')))
print len(queued_feeds)
if len(queued_feeds)>0:
r.sadd('queued_feeds', *queued_feeds)
logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
len(queued_feeds),
r.zcard('tasked_feeds'),
r.scard('queued_feeds'),
r.zcard('scheduled_updates')))
# Regular feeds
if tasked_feeds_size < 5000:
feeds = r.srandmember('queued_feeds',5000)
Feed.task_feeds(feeds, verbose=True)
active_count = len(feeds)
else:
logging.debug(" ---> ~SN~FBToo many tasked feeds. ~SB%s~SN tasked." % tasked_feeds_size)
active_count = 0
cp1 = time.time()
# Force refresh feeds
refresh_feeds = Feed.objects.filter(
active=True,
fetched_once=False,
active_subscribers__gte=1
).order_by('?')[:100]
refresh_count = refresh_feeds.count()
cp2 = time.time()
# Mistakenly inactive feeds
hours_ago = (now - datetime.timedelta(minutes=10)).strftime('%s')
old_tasked_feeds = r.zrangebyscore('tasked_feeds', 0, hours_ago)
inactive_count = len(old_tasked_feeds)
if inactive_count:
r.zremrangebyscore('tasked_feeds', 0, hours_ago)
# r.sadd('queued_feeds', *old_tasked_feeds)
for feed_id in old_tasked_feeds:
r.zincrby('error_feeds', feed_id, 1)
feed = Feed.get_by_id(feed_id)
feed.set_next_scheduled_update()
logging.debug(" ---> ~SN~FBRe-queuing ~SB%s~SN dropped feeds (~SB%s/%s~SN queued/tasked)" % (
inactive_count,
r.scard('queued_feeds'),
r.zcard('tasked_feeds')))
cp3 = time.time()
old = now - datetime.timedelta(days=1)
old_feeds = Feed.objects.filter(
next_scheduled_update__lte=old,
active_subscribers__gte=1
).order_by('?')[:500]
old_count = old_feeds.count()
cp4 = time.time()
logging.debug(" ---> ~FBTasking ~SB~FC%s~SN~FB/~FC%s~FB (~FC%s~FB/~FC%s~SN~FB) feeds... (%.4s/%.4s/%.4s/%.4s)" % (
active_count,
refresh_count,
inactive_count,
old_count,
cp1 - start,
cp2 - cp1,
cp3 - cp2,
cp4 - cp3
))
Feed.task_feeds(refresh_feeds, verbose=True)
Feed.task_feeds(old_feeds, verbose=True)
logging.debug(" ---> ~SN~FBTasking took ~SB%s~SN seconds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
int((time.time() - start)),
r.zcard('tasked_feeds'),
r.scard('queued_feeds'),
r.zcard('scheduled_updates')))
except Exception, e:
import traceback
traceback.print_exc()
logging.error(str(e))
if settings.SEND_ERROR_MAILS:
mail_admins("Error in Task-Feeds",str(e)+'\n'+traceback.format_exc())
def export_xls_full(raw):
"""
Выгрузка полной (с внутренними номерами) статистики звонков
:param raw:
:return:
"""
wb = xlwt.Workbook()
ws = wb.add_sheet('Подробный список')
# Заголовок
ws.write(0, 0, 'Гор. номер')
ws.write_merge(0, 0, 1, 5, 'Вх.')
ws.write_merge(0, 0, 6, 10, 'Исх.')
path = get_options('main', 'xls_path_full', True)
if not path:
log.critical('Ошибка чтения конфигурационного файла, см. ошибки выше')
return
line = 1
for kc in sorted(raw):
ws.write(line, 0, kc)
ki = raw[kc]['inc']
ko = raw[kc]['out']
kiu = ki['users']
kou = ko['users']
ws.write(line, 2, format_time(ki['duration']))
ws.write(line, 3, ki['count'])
ws.write(line, 4, format_time(ki['billsec']))
ws.write(line, 5, ki['answer'])
ws.write(line, 7, format_time(ko['duration']))
ws.write(line, 8, ko['count'])
ws.write(line, 9, format_time(ko['billsec']))
ws.write(line, 10, ko['answer'])
inc_line = 0
for inc in sorted(kiu):
inc_line += 1
ws.write(line + inc_line, 1, inc)
ws.write(line + inc_line, 4, format_time(kiu[inc]['billsec']))
ws.write(line + inc_line, 5, kiu[inc]['answer'])
out_line = 0
for out in sorted(kou):
out_line += 1
ws.write(line + out_line, 6, out)
ws.write(line + out_line, 7, format_time(kou[out]['duration']))
ws.write(line + out_line, 8, kou[out]['count'])
ws.write(line + out_line, 9, format_time(kou[out]['billsec']))
ws.write(line + out_line, 10, kou[out]['answer'])
line += max([inc_line, out_line]) + 1
try:
wb.save(path)
except PermissionError as e:
log.error('Недостаточно прав для сохранения файла: %s' % e.filename)
return
except FileNotFoundError as e:
log.error('Неверный путь или имя файла: %s' % e.filename)
return
return True
def copy_to_my_pan(self, user_id, user_ref_id, share_log: ShareLogs,
default_pan_id):
fs_id = share_log.fs_id
key_prefix = "client:ready:"
# 文件已迁出,等待迁入
async_service.update_state(key_prefix, user_id, {"state": 0, "pos": 1})
top_dir_item = self.checkout_root_item(user_id, user_ref_id,
default_pan_id)
if not top_dir_item:
return -3, None # 无法创建顶级目录
pan_acc: PanAccounts = auth_service.get_pan_account(
default_pan_id, user_id)
if not pan_acc:
logger.error("PanAccount not exists![{}],user_id:{}".format(
default_pan_id, user_id))
return -3, None
pan_acc = auth_service.check_pan_token_validation(pan_acc)
# 目录结构构建
async_service.update_state(key_prefix, user_id, {"state": 0, "pos": 2})
find_pan_file = False
client_item_params = {}
jsonrs = restapi.file_search(pan_acc.access_token,
key=fs_id,
parent_dir=top_dir_item.path)
if jsonrs:
for finfo in jsonrs:
if "fs_id" in finfo:
md5_val = finfo["md5"]
if md5_val == share_log.md5_val:
find_pan_file = True
client_item_params = self.build_client_item_params(
finfo, top_dir_item.id, user_ref_id, pan_acc.id,
fs_id)
break
if find_pan_file:
client_data_item = ClientDataDao.new_data_item(client_item_params)
return 0, client_data_item
# print("target pan acc id:", pan_acc.id)
jsonrs = restapi.transfer_share_files(pan_acc.access_token,
share_id=share_log.share_id,
from_uk=share_log.uk,
randsk=share_log.randsk,
fs_id=share_log.fs_id,
path=top_dir_item.path)
if "extra" in jsonrs and "list" in jsonrs["extra"]:
file_path = jsonrs["extra"]["list"][0]["to"]
key = self.parse_query_key(file_path)
# 迁出后, 获取信息
async_service.update_state(key_prefix, user_id, {
"state": 0,
"pos": 3
})
time.sleep(3)
client_data_item = self.check_file_by_key_search(
key, top_dir_item.path, top_dir_item.id, share_log.md5_val,
fs_id, user_ref_id, pan_acc)
if client_data_item:
return 0, client_data_item
return -4, None # 无法转存文件
elif jsonrs.get('errno', 0) == -30 and 'path' in jsonrs:
logger.info("exists jsonrs:{}".format(jsonrs))
file_path = jsonrs.get('path', None)
if file_path:
# 迁出后, 获取信息
async_service.update_state(key_prefix, user_id, {
"state": 0,
"pos": 3
})
key = self.parse_query_key(file_path)
client_data_item = self.check_file_by_key_search(
key, top_dir_item.path, top_dir_item.id, share_log.md5_val,
fs_id, user_ref_id, pan_acc)
if client_data_item:
return 0, client_data_item
return -4, None # 无法转存文件
elif "errno" in jsonrs:
return -4, None # 无法转存文件
def fit(self,
matrix,
k,
distance,
shrink=0,
threshold=0,
implicit=True,
alpha=None,
beta=None,
l=None,
c=None,
verbose=False):
"""
Initialize the model and compute the similarity matrix S with a distance metric.
Access the similarity matrix using: self._sim_matrix
Parameters
----------
matrix : csr_matrix
A sparse matrix. For example, it can be the URM of shape (number_users, number_items).
k : int
K nearest neighbour to consider.
distance : str
One of the supported distance metrics, check collaborative_filtering_base constants.
shrink : float, optional
Shrink term used in the normalization
threshold: float, optional
All the values under this value are cutted from the final result
implicit: bool, optional
If true, treat the URM as implicit, otherwise consider explicit ratings (real values) in the URM
alpha: float, optional, included in [0,1]
beta: float, optional, included in [0,1]
l: float, optional, balance coefficient used in s_plus distance, included in [0,1]
c: float, optional, cosine coefficient, included in [0,1]
"""
alpha = -1 if alpha is None else alpha
beta = -1 if beta is None else beta
l = -1 if l is None else l
c = -1 if c is None else c
if distance == self.SIM_ASYMCOSINE and not (0 <= alpha <= 1):
log.error(
'Invalid parameter alpha in asymmetric cosine similarity!')
return
if distance == self.SIM_TVERSKY and not (0 <= alpha <= 1
and 0 <= beta <= 1):
log.error('Invalid parameter alpha/beta in tversky similarity!')
return
if distance == self.SIM_P3ALPHA and alpha is None:
log.error('Invalid parameter alpha in p3alpha similarity')
return
if distance == self.SIM_RP3BETA and alpha is None and beta is None:
log.error('Invalid parameter alpha/beta in rp3beta similarity')
return
if distance == self.SIM_SPLUS and not (0 <= l <= 1 and 0 <= c <= 1
and 0 <= alpha <= 1
and 0 <= beta <= 1):
log.error('Invalid parameter alpha/beta/l/c in s_plus similarity')
return
# compute and stores the similarity matrix using one of the distance metric: S = R•R'
if distance == self.SIM_COSINE:
self._sim_matrix = sim.cosine(matrix,
k=k,
shrink=shrink,
threshold=threshold,
binary=implicit)
elif distance == self.SIM_ASYMCOSINE:
self._sim_matrix = sim.asymmetric_cosine(matrix,
k=k,
shrink=shrink,
threshold=threshold,
binary=implicit,
alpha=alpha)
elif distance == self.SIM_JACCARD:
self._sim_matrix = sim.jaccard(matrix,
k=k,
shrink=shrink,
threshold=threshold,
binary=implicit)
elif distance == self.SIM_DICE:
self._sim_matrix = sim.dice(matrix,
k=k,
shrink=shrink,
threshold=threshold,
binary=implicit)
elif distance == self.SIM_TVERSKY:
self._sim_matrix = sim.tversky(matrix,
k=k,
shrink=shrink,
threshold=threshold,
binary=implicit,
alpha=alpha,
beta=beta)
elif distance == self.SIM_P3ALPHA:
self._sim_matrix = sim.p3alpha(matrix,
k=k,
shrink=shrink,
threshold=threshold,
binary=implicit,
alpha=alpha)
elif distance == self.SIM_RP3BETA:
self._sim_matrix = sim.rp3beta(matrix,
k=k,
shrink=shrink,
threshold=threshold,
binary=implicit,
alpha=alpha,
beta=beta)
elif distance == self.SIM_SPLUS:
self._sim_matrix = sim.s_plus(matrix,
k=k,
shrink=shrink,
threshold=threshold,
binary=implicit,
l=l,
t1=alpha,
t2=beta,
c=c)
else:
log.error('Invalid distance metric: {}'.format(distance))
#self.SIM_DOTPRODUCT: sim.dot_product(matrix, k=k, shrink=shrink, threshold=threshold, binary=implicit)
return self._sim_matrix
use_salience=opts.use_salience,
salience_features=salience_features)
else:
assert model.pair_composition_network, \
'pair_composition_network in the model cannot be None'
assert model.pair_composition_network.use_salience \
== opts.use_salience
assert model.pair_composition_network.salience_features \
== salience_features
trainer = EventCompositionTrainer(model,
saving_path=opts.output_path,
log=log)
if not os.path.isdir(opts.indexed_corpus):
log.error('Cannot find indexed corpus at {}'.format(
opts.indexed_corpus))
exit(-1)
log.info('Loading indexed corpus from: {}, with batch_size={}, '
'use_salience={}'.format(opts.indexed_corpus, opts.batch_size,
opts.use_salience))
corpus_it = PairTuningCorpusIterator(
opts.indexed_corpus,
batch_size=opts.batch_size,
use_salience=opts.use_salience,
salience_features=salience_features)
log.info('Found {} lines in the corpus'.format(len(corpus_it)))
val_corpus_it = None
if opts.val_indexed_corpus and os.path.isdir(opts.val_indexed_corpus):
log.info('Loading validation indexed corpus from: {}, '
def analyse(self):
if self.directory is None:
log.critical("Please set directory")
sys.exit()
log.info('Start code static analyse...')
d = directory.Directory(self.directory)
files = d.collect_files()
log.info('Scan Files: {0}, Total Time: {1}s'.format(
files['file_nums'], files['collect_time']))
ext_language = {
# Image
'.jpg': 'image',
'.png': 'image',
'.bmp': 'image',
'.gif': 'image',
'.ico': 'image',
'.cur': 'image',
# Font
'.eot': 'font',
'.otf': 'font',
'.svg': 'font',
'.ttf': 'font',
'.woff': 'font',
# CSS
'.css': 'css',
'.less': 'css',
'.scss': 'css',
'.styl': 'css',
# Media
'.mp3': 'media',
'.swf': 'media',
# Execute
'.exe': 'execute',
'.sh': 'execute',
'.dll': 'execute',
'.so': 'execute',
'.bat': 'execute',
'.pl': 'execute',
# Edit
'.swp': 'tmp',
# Cert
'.crt': 'cert',
# Text
'.txt': 'text',
'.csv': 'text',
'.md': 'markdown',
# Backup
'.zip': 'backup',
'.bak': 'backup',
'.tar': 'backup',
'.rar': 'backup',
'.tar.gz': 'backup',
'.db': 'backup',
# Config
'.xml': 'config',
'.yml': 'config',
'.spf': 'config',
'.iml': 'config',
'.manifest': 'config',
# Source
'.psd': 'source',
'.as': 'source',
# Log
'.log': 'log',
# Template
'.template': 'template',
'.tpl': 'template',
}
for ext in files:
if ext in ext_language:
log.info('{0} - {1}'.format(ext, files[ext]))
continue
else:
log.info(ext)
languages = CobraLanguages.query.all()
rules = CobraRules.query.filter_by(status=1).all()
extensions = None
for rule in rules:
for language in languages:
if language.id == rule.language:
extensions = language.extensions.split('|')
if extensions is None:
log.warning("Rule Language Error")
# grep name is ggrep on mac
grep = '/bin/grep'
if 'darwin' == sys.platform:
log.info('In Mac OS X System')
for root, dir_names, file_names in os.walk(
'/usr/local/Cellar/grep'):
for filename in file_names:
if 'ggrep' == filename:
grep = os.path.join(root, filename)
filters = []
for e in extensions:
filters.append('--include=*' + e)
# White list
white_list = []
ws = CobraWhiteList.query.filter_by(project_id=self.project_id,
rule_id=rule.id,
status=1).all()
if ws is not None:
for w in ws:
white_list.append(w.path)
try:
log.info('Scan rule id: {0}'.format(rule.id))
# -n Show Line number / -r Recursive / -P Perl regular expression
p = subprocess.Popen([grep, "-n", "-r", "-P"] + filters +
[rule.regex, self.directory],
stdout=subprocess.PIPE)
result = p.communicate()
# Exists result
if len(result[0]):
log.info('Found:')
per_line = str(result[0]).split("\n")
log.debug(per_line)
for r in range(0, len(per_line) - 1):
try:
rr = str(per_line[r]).replace(self.directory,
'').split(':', 1)
code = str(rr[1]).split(':', 1)
if self.task_id is None:
self.task_id = 0
rule_id = rule.id
current_time = datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
m_file = rr[0].strip()
m_line = code[0]
m_code = str(code[1].strip())
params = [
self.task_id, rule_id, m_file, m_line, m_code,
current_time, current_time
]
try:
if m_file in white_list or ".min.js" in m_file:
log.debug("In White list or min.js")
else:
# # // /* *
match_result = re.match(
"(#)?(//)?(\*)?(/\*)?", m_code)
if match_result.group(
0
) is not None and match_result.group(
0) is not "":
log.debug("In Annotation")
else:
log.debug('In Insert')
if rule.regex == "":
# Didn't filter line when regex is empty
r_content = CobraResults.query.filter_by(
task_id=self.task_id,
rule_id=rule_id,
file=m_file).first()
m_line = 0
else:
r_content = CobraResults.query.filter_by(
task_id=self.task_id,
rule_id=rule_id,
file=m_file,
line=m_line).first()
if r_content is not None:
log.warning("Exists Result")
else:
results = CobraResults(
self.task_id, rule_id, m_file,
m_line, m_code, current_time,
current_time)
db.session.add(results)
db.session.commit()
log.info('Insert Results Success')
except Exception as e:
log.error('Insert Results Failed' +
str(e.message))
log.debug(params)
except Exception as e:
log.critical('Error parsing result: ' +
str(e.message))
else:
log.info('Not Found')
except Exception as e:
log.critical('Error calling grep: ' + str(e))
# Set End Time For Task
t = CobraTaskInfo.query.filter_by(id=self.task_id).first()
t.status = 2
t.file_count = files['file_nums']
t.time_end = int(time.time())
t.time_consume = t.time_end - t.time_start
t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
try:
db.session.add(t)
db.session.commit()
except Exception as e:
log.critical("Set start time failed:" + e.message)
log.info("Scan Done")
def _handle_ctrl_msg(self, m):
"""Handle all types of CTRL messages.
May change inner state (see diagram).
@m - a CTRL message
"""
# switch case on msg type
typ = m.get_type()
# Reset Node
if typ == Message.TYPE.Reset:
self._io.reset()
self.reset()
self._state = self.State.UNINITIALIZED
# Send debug data to Master
elif typ == Message.TYPE.DebugData:
self._send_to_master(Message.TYPE.DebugData, repr(self))
# Read and set given serialized circuit
elif typ == Message.TYPE.SetCircuit:
self._assert_state("SetCircuit", self.State.UNINITIALIZED)
if self._circuit is not None:
log.warn("Setting new circuit without reset!")
c = unserialize(m.get_msg(), Circuit)
self._set_circuit(c)
if self._fully_initialized():
self._state = self.State.INITIALIZED
# Read and set given parameters for secret sharing schemes
elif typ == Message.TYPE.SetSecretSharing:
self._assert_state("SetSecretSharing", self.State.UNINITIALIZED)
ss_args, ss2_args = unserialize(m.get_msg())
self._set_secret_sharing(ss_args, ss2_args)
if self._fully_initialized():
self._state = self.State.INITIALIZED
# Read and set given input as party's input (FFE)
elif typ == Message.TYPE.SetInput:
self._assert_state("SetInput", self.State.UNINITIALIZED)
secret_input = unserialize(m.get_msg(), FFE)
self._set_input(secret_input)
if self._fully_initialized():
self._state = self.State.INITIALIZED
# Set party's input to a random input in the given serialized field
elif typ == Message.TYPE.SetRandInput:
self._assert_state("SetRandInput", self.State.UNINITIALIZED)
field = unserialize(m.get_msg())
secret_input = field.rand()
log.info("Node %d: Setting my input to %r (random)", self._id,
secret_input)
self._set_input(secret_input)
if self._fully_initialized():
self._state = self.State.INITIALIZED
# Share party's input. Node must INITIALIZED (input and circuit received)
elif typ == Message.TYPE.ShareInput:
self._assert_state("ShareInput", self.State.INITIALIZED)
self._input_shares = [None] * self._n_parties
self._share(self._ss, self._input, Message.TYPE.InputShare)
self._state = self.State.RECEIVING_INPUTS
# Read and set given Truncinator as party's truncinator (for multiplication)
elif typ == Message.TYPE.SetTruncinator:
self._assert_state("SetTruncinator", [
self.State.UNINITIALIZED, self.State.INITIALIZED,
self.State.RUNNING
])
msg = m.get_msg()
self._truncinator = unserialize(msg, Truncinator)
# Read and set given Resampler as party's resampler (for bulk random masks)
elif typ == Message.TYPE.SetResampler:
self._assert_state("SetResampler", [
self.State.UNINITIALIZED, self.State.INITIALIZED,
self.State.RUNNING
])
msg = m.get_msg()
self._resampler = unserialize(msg, Resampler)
# Generate a bulk of masks for DIK10 multiplication
elif typ == Message.TYPE.GenBulkMulMasks:
self._assert_state("GenBulkMulMasks",
[self.State.INITIALIZED, self.State.RUNNING])
assert self._resampler is not None, "Cannot generate bulk masks - Resampler not set!"
rand_val = self._field.rand()
self._share(self._ss, rand_val, Message.TYPE.DIKBulkShareLow)
self._share(self._ss2, rand_val, Message.TYPE.DIKBulkShareHigh)
self._dik_low_shares = [None] * self._n_parties
self._dik_high_shares = [None] * self._n_parties
self._old_state = self._state
self._state = self.State.GEN_BULK_MUL_MASKS
# Generation of a bulk of masks for node going down - Phase1
elif typ == Message.TYPE.GenBulkRegenMasks:
self._assert_state("GenBulkRegenMasks",
[self.State.INITIALIZED, self.State.RUNNING])
assert self._resampler is not None, "Cannot generate bulk masks - Resampler not set!"
# Choose a random polynomial with zero in the position to be regenerated & send shares
self._regen_pos = unserialize(m.get_msg())
zero = self._field.zero()
self._share(self._ss,
zero,
Message.TYPE.RegenRandPolyShare,
pos=self._regen_pos)
# Init structures for collecting shares and masks
self._regen_shares = [None] * self._n_parties
if self._regen_pos not in self._regen_masks:
self._regen_masks[self._regen_pos] = []
self._old_state = self._state
self._state = self.State.GEN_BULK_REGEN_MASKS
# Evaluate a linear gate. Gate's name received in message
elif typ == Message.TYPE.EvalGate:
self._assert_state("EvalGate", self.State.RUNNING)
self._eval_linear_gate(m.get_msg())
### <DIK multiplication> ###
# Initialize and start evaluation of multiplication gate. Gate's name received in message
elif typ == Message.TYPE.EvalMulGateDIK:
self._assert_state("EvalMulGateDIK", self.State.RUNNING)
assert len(self._dik_low) == len(
self._dik_high
), "DIK sanity check fail - low & high masks out of sync!"
assert self._dik_ptr < len(
self._dik_low
), "Cannot multiply DIK - Not enough masks generated!"
# Parse message for gate's name and the party performing the calculation
self._mul_gate, self._mul_party = unserialize(m.get_msg())
# Calculate local multiplication, add current (high degree) random mask & send to "dealer"
naive_mult = self._circuit.evaluate_gate(self._mul_gate)
masked_mult = self._dik_high[self._dik_ptr] + naive_mult
processed_share = self._ss2.preprocess(
masked_mult, self._id, pos=None) # pos=None for secret pos
self._send_to_party(self._mul_party, Message.TYPE.DIKMulHighShare,
processed_share)
# Am I the dealer?
if self._id == self._mul_party:
self._state = self.State.MUL_DIK_DEALER
self._mul_shares = [None] * self._n_parties
else:
self._state = self.State.MUL_DIK_NON_DEALER
### </DIK multiplication> ###
### <BGW multiplication> ###
# Initialize and start evaluation of multiplication gate. Gate's name received in message
elif typ == Message.TYPE.EvalMulGateInit:
self._assert_state("EvalMulGateInit", self.State.RUNNING)
assert self._truncinator is not None, "Cannot multiply - Truncinator not set!"
# Init data structures for multiplication
self._mul_gate = m.get_msg()
self._mul_shares = [None] * self._n_parties
# Generate a my part in the random mask & share
zero = self._field.zero()
self._share(self._ss2, zero, Message.TYPE.BGWMulRandPolyShare)
self._state = self.State.MUL_RANDPOLY_SHARE
# Perform local multiplication and mask result using the joint mask by received shares
elif typ == Message.TYPE.EvalMulGateReduce:
self._assert_state("EvalMulGateReduce",
self.State.MUL_RANDPOLY_DONE)
# Calculate regular multiplication and add joint random mask
naive_mult = self._circuit.evaluate_gate(self._mul_gate)
masked_mult = sum(self._mul_shares, naive_mult)
# Share result (in high dimension) & reset data structure for new shares
self._share(self._ss2, masked_mult,
Message.TYPE.BGWMulReductionShare)
self._mul_shares = [None] * self._n_parties
self._state = self.State.MUL_REDUCTION_SHARE
# Perform degree reduction on shares received, and send each party back its appropriate part
elif typ == Message.TYPE.EvalMulGateFinalize:
self._assert_state("EvalMulGateFinalize",
self.State.MUL_REDUCTION_CALC)
# Calculate linear degree reduction on the vector of evaluations
reduced = self._truncinator.reduce(self._mul_shares)
# Send back the reduced result & reset data structure for new shares
processed_shares = [
self._ss2.preprocess(s, self._id, pos=None) for s in reduced
]
self._send_to_parties(Message.TYPE.BGWMulReductionResult,
processed_shares)
self._mul_shares = [None] * self._n_parties
self._state = self.State.MUL_RECONSTRUCTION
### </BGW multiplication> ###
# Send my final output to all of the parties
elif typ == Message.TYPE.EvalOutput:
self._assert_state("ShareOutput", self.State.RUNNING)
self._output_shares = [None] * self._n_parties
out_share = self._circuit.get_output()
output_msg = self._ss.preprocess(
out_share, self._id, pos=None) # pos=None for secret pos
msgs = [output_msg] * self._n_parties
self._send_to_parties(Message.TYPE.OutputShare, msgs)
self._state = self.State.RECEIVING_OUTPUTS
else:
log.error("Unknown ctrl msg received:\n%r", m)
def wizard_hybrid():
SIM_MATRIX = ['saved_sim_matrix', 'saved_sim_matrix_evaluation']
R_HAT = ['saved_r_hat', 'saved_r_hat_evaluation']
SAVE = ['saved_sim_matrix', 'saved_r_hat']
EVALUATE = ['saved_sim_matrix_evaluation', 'saved_r_hat_evaluation']
start = time.time()
matrices_array, folder, models = hb.create_matrices_array()
print('matrices loaded in {:.2f} s'.format(time.time() - start))
log.success('You have loaded: {}'.format(models))
NORMALIZATION_MODE = normalization_mode_selection()
if folder in SAVE:
WEIGHTS = weights_selection(models)
if folder in SIM_MATRIX:
name, urm_filter_tracks, rel_path = option_selection_save('SIM')
hybrid_rec = HybridSimilarity(
matrices_array,
normalization_mode=NORMALIZATION_MODE,
urm_filter_tracks=urm_filter_tracks)
sps.save_npz('raw_data/' + rel_path + name,
hybrid_rec.get_r_hat(weights_array=WEIGHTS))
if folder in R_HAT:
name, urm_filter_tracks, rel_path, EXPORT = option_selection_save(
'R_HAT')
hybrid_rec = HybridRHat(matrices_array,
normalization_mode=NORMALIZATION_MODE,
urm_filter_tracks=urm_filter_tracks)
if EXPORT:
N = ask_number_recommendations()
recommendations = hybrid_rec.recommend_batch(
weights_array=WEIGHTS,
target_userids=data.get_target_playlists(),
N=N)
exportcsv(recommendations, path='submission', name=name)
else:
sps.save_npz('raw_data/' + rel_path + name,
hybrid_rec.get_r_hat(weights_array=WEIGHTS))
elif folder in EVALUATE:
log.success('|WHAT YOU WANT TO DO ???|')
log.warning('\'1\' BAYESIAN SEARCH VALIDATION')
log.warning('\'2\' HAND CRAFTED WEIGHTS')
mode = input()[0]
# BAYESIAN SEARCH
if mode == '1':
log.success(
'|SELECT A NUMBER OF |||ITERATIONS||| FOR THE ALGORITHM|')
iterations = float(input())
urm_filter_tracks = data.get_urm_train_1()
if folder in SIM_MATRIX:
hybrid_rec = HybridSimilarity(
matrices_array,
normalization_mode=NORMALIZATION_MODE,
urm_filter_tracks=urm_filter_tracks)
if folder in R_HAT:
hybrid_rec = HybridRHat(matrices_array,
normalization_mode=NORMALIZATION_MODE,
urm_filter_tracks=urm_filter_tracks)
hybrid_rec.validate(iterations=iterations,
urm_test=data.get_urm_test_1(),
userids=data.get_target_playlists())
# MANUAL WEIGHTS
elif mode == '2':
WEIGHTS = weights_selection(models)
urm_filter_tracks = data.get_urm_train_1()
chose = option_selection_evaluation_2() # save, evaluate or csv
if chose == 's':
log.success('|CHOSE A NAME FOR THE MATRIX...|')
name = input()
if folder in SIM_MATRIX:
type = 'SIM'
hybrid_rec = HybridSimilarity(
matrices_array,
normalization_mode=NORMALIZATION_MODE,
urm_filter_tracks=urm_filter_tracks)
elif folder in R_HAT:
type = 'R_HAT'
hybrid_rec = HybridRHat(
matrices_array,
normalization_mode=NORMALIZATION_MODE,
urm_filter_tracks=urm_filter_tracks)
sps.save_npz('raw_data/saved_r_hat_evaluation/' + name,
hybrid_rec.get_r_hat(weights_array=WEIGHTS))
sym_rec = symmetric_recommender_creator(
models,
type,
NORMALIZATION_MODE,
urm_filter_tracks=data.get_urm_train_2())
sps.save_npz('raw_data/saved_r_hat_evaluation_2/' + name,
sym_rec.get_r_hat(weights_array=WEIGHTS))
elif chose == 'e':
if folder in SIM_MATRIX:
type = 'SIM'
hybrid_rec = HybridSimilarity(
matrices_array,
normalization_mode=NORMALIZATION_MODE,
urm_filter_tracks=urm_filter_tracks)
elif folder in R_HAT:
type = 'R_HAT'
hybrid_rec = HybridRHat(
matrices_array,
normalization_mode=NORMALIZATION_MODE,
urm_filter_tracks=urm_filter_tracks)
N = ask_number_recommendations()
print('Recommending...')
recs = hybrid_rec.recommend_batch(
weights_array=WEIGHTS,
target_userids=data.get_target_playlists(),
N=N)
hybrid_rec.evaluate(recommendations=recs,
test_urm=data.get_urm_test_1())
# export the recommendations
log.success(
'Do you want to save the CSV with these recomendations? (y/n)'
)
if input()[0] == 'y':
export_csv_wizard(recs)
sym_rec = symmetric_recommender_creator(
models,
type,
NORMALIZATION_MODE,
urm_filter_tracks=data.get_urm_train_2())
recs2 = sym_rec.recommend_batch(
weights_array=WEIGHTS,
target_userids=data.get_target_playlists())
sym_rec.evaluate(recommendations=recs2,
test_urm=data.get_urm_test_2())
elif chose == 'c':
if folder in R_HAT:
hybrid_rec = HybridRHat(
matrices_array,
normalization_mode=NORMALIZATION_MODE,
urm_filter_tracks=urm_filter_tracks)
N = ask_number_recommendations()
print('Recommending...')
recs = hybrid_rec.recommend_batch(
weights_array=WEIGHTS,
target_userids=data.get_target_playlists(),
N=N)
export_csv_wizard(recs)
else:
log.error('not implemented yet')
else:
log.error('WRONG FOLDER')
def get_at_inc_list():
"""
Импортируем список городских входящих номеров из БД АТС
:return: {string: {string}}, {гор_номер: {вн_номер, ...}}
"""
raw = defaultdict(set)
options_list = get_options('asterisk', 'db')
if options_list:
host, user, password, db = options_list
try:
with pymysql.connect(host, user, password, db) as cur:
re_group = re.compile(r'ext-group,(\d{3}),1')
re_im = re.compile(r'from-did-direct,(\d{4}),1')
re_ivr = re.compile(r'ivr-(\d+),s,1')
# Запрос, привязка групп к вн. номерам
cur.execute("SELECT grpnum, grplist FROM ringgroups ")
# Составляем словарь {группа: set(список_вн_номеров)}
groups_list = dict(
(k, v.replace('#', '').split('-')) for k, v in cur)
cur.execute(
"SELECT ivr_id, selection, dest FROM ivr_dests WHERE ivr_ret = 0"
)
ivr_list = defaultdict(dict)
for ivr_id, sel, dest in cur:
ivr_group = re_group.match(dest)
if ivr_group:
ivr_list[str(ivr_id)][sel] = groups_list[
ivr_group.group(1)]
else:
ivr_im = re_im.match(dest)
if ivr_im:
ivr_list[str(ivr_id)][sel] = [ivr_im.group(1)]
# Запрос, привязка гор. номеров к вн. номерам или группам
cur.execute(
"SELECT extension, destination FROM incoming "
"WHERE LENGTH(extension) = 7 OR LENGTH(extension) = 11")
for ext, des in cur:
# Формат гор. номера "###-##-##"
cm = '%s-%s-%s' % (ext[-7:-4], ext[-4:-2], ext[-2:])
# Выбираем только группы
group = re_group.match(des)
if group:
for x in groups_list[group.group(1)]:
# Добавляем все вн. номера группы связанные с гор. номером
raw[cm].add(x)
else:
# Выбираем только прямые вн. номера
im = re_im.match(des)
if im:
# Добавляем вн. номер связанный с гор. номером
# raw[im.group(1)].add(cm)
raw[cm].add(im.group(1))
else:
ivr = re_ivr.match(des)
if ivr and ivr.group(1) in ivr_list:
for ik, iv in ivr_list[ivr.group(1)].items():
for x in iv:
ivr_cm = '%s/%s' % (cm, ik)
raw[ivr_cm].add(x)
except OperationalError as e:
log.error(e)
return raw
step_cmds.append(base_cmd + f" {poolname}")
step_cmds.append(base_cmd + f" -p {poolname}")
step_cmds.append(base_cmd + f" --pool {poolname}")
for step in step_cmds:
rbd_util.exec_cmd(f"rbd mirror pool enable {poolname} pool")
if rbd_util.exec_cmd(step) == False:
log.error(f"Test case Failed executing: {step}")
exit(1)
if not rbd_util.exec_cmd(
f'rbd mirror pool info {poolname}|grep \\"Mode: disabled\\"'):
log.error(f"command not worked: {step}")
exit(1)
log.info("Test Case Passed")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="RBD CLI Test")
parser.add_argument("-e", "--ec-pool-k-m", required=False)
parser.add_argument("--test-case", required=True)
args = parser.parse_args()
try:
globals()[args.test_case]()
rbd_util.delete_pool(poolname=args.test_case)
except KeyError:
log.error(f"{args.test_case} not yet implemented")
try:
api_event = APIEventOps(**config_data)
api_event.get_event()
api_event_cluster_event = APIClusterEventOps(**config_data)
api_event_cluster_event.get_cluster_event()
api_server_event = APIServerEventOps(**config_data)
api_server_event.get_server_event_api()
add_test_info.status("ok")
except Exception, e:
log.error("test error")
log.error(e)
add_test_info.status("error")
add_test_info.completed_info()
if __name__ == "__main__":
config_data = config.get_config()
if not config_data["auth"]:
log.error("auth failed")
else:
exec_test(config_data)
def update_translation():
"""1. 更新翻译"""
NEED_CLEAR = True
log.info(os.getcwd())
eso_path = ' '.join(sys.argv[2:])
log.info('ESO PATH: %s' % eso_path)
mnf_path = os.path.join(eso_path, 'depot/eso.mnf')
extract_path = '../../temp/extract/'
datestr = time.strftime('%Y%m%d')
print('请输入文件后缀并按下回车: (默认为今天日期: %s)' % datestr)
datestr_new = input().strip()
if datestr_new != '':
datestr = datestr
log.info('filename suffix: %s' % datestr)
if NEED_CLEAR:
print('### 正在清理...')
log.debug('正在清理...')
# 清理输出目录
dirs = ('../../输出/更新翻译/1_new/',
'../../输出/更新翻译/2_diff/',)
for dir in dirs:
if os.path.exists(dir):
log.info('clear %s' % dir)
shutil.rmtree(dir)
# 清理 mnf 提取目录
if os.path.exists(extract_path):
log.info('clear %s' % extract_path)
shutil.rmtree(extract_path)
# 清理翻译中间目录
log.info('clear csv and xlsx')
for root, dirs, files in os.walk('../../translation/lang'):
for f in files:
if (f.startswith('en.') or f.startswith('jp.')) \
and (f.endswith('.lang.csv') or f.endswith('.lang.xlsx')):
filename = os.path.join(root, f)
log.debug('remove %s' % filename)
os.remove(filename)
break
log.info('clear ui text')
files = (
'../../translation/en_pregame.lua',
'../../translation/en_client.lua',
'../../translation/zh_translate.txt',
)
for f in files:
if os.path.exists(f):
log.debug('remove %s' % f)
os.remove(f)
print('### 创建目录...')
log.debug('创建目录...')
dirs = (
'../../输出/更新翻译/1_new/',
'../../输出/更新翻译/2_diff/',
'../../输出/更新翻译/4_old/',
)
for dir in dirs:
if not os.path.isdir(dir):
log.info('create %s' % dir)
os.makedirs(dir)
print('### 正在解码...')
log.debug('正在解码...')
log.debug('extract eso.mnf -a 0')
execute('EsoExtractData.exe "%s" -a 0 ../../temp/extract' % mnf_path)
log.debug('extract eso.mnf -a 2')
execute('EsoExtractData.exe "%s" -a 2 ../../temp/extract' % mnf_path)
if not os.path.exists(extract_path):
log.error('提取失败')
sys.exit(-1)
os.chdir('../../')
log.info(os.getcwd())
print('### 正在复制...')
log.debug('正在复制...')
src_dst = (
('temp/extract/gamedata/lang/en.lang.csv', 'translation/lang/en.lang.csv',),
('temp/extract/gamedata/lang/jp.lang.csv', 'translation/lang/jp.lang.csv',),
('temp/extract/esoui/lang/en_pregame.lua', 'translation/en_pregame.lua',),
('temp/extract/esoui/lang/en_client.lua', 'translation/en_client.lua',),
)
for src, dst in src_dst:
log.info('copy %s to %s' % (src, dst))
shutil.copy(src, dst)
os.chdir('scripts/')
log.info(os.getcwd())
print('### 正在分析...')
log.debug('正在分析...')
execute('python split_lang_csv_by_id.py')
execute('python split_lang_csv_by_id.py -l jp')
print('### 正在导出新版xls...')
log.debug('正在导出新版xls...')
execute('python prepare_lang.py --all')
execute('python convert_lua_to_txt.py')
execute('python convert_txt_to_xls.py')
print('### 正在保存结果...')
log.debug('正在保存结果...')
dst = '../输出/更新翻译/1_new/'
for root, dirs, files in os.walk('../translation/lang'):
for f in files:
if f.startswith('en.') and f.endswith('.lang.xlsx'):
filename = os.path.join(root, f)
log.info('copy %s to %s' % (filename, dst))
shutil.copy(filename, dst)
break
filename = '../translation/zh_translate.xlsx'
log.info('copy %s to %s' % (filename, dst))
shutil.copy(filename, dst)
print('### 正在重命名...')
log.debug('正在重命名...')
execute('python rename_lang_xls.py %s %s' % (datestr, dst))
def query_history(self, req: HistoryRequest) -> Optional[List[BarData]]:
"""
从tushare里查询历史数据
:param req:查询请求
:return: Optional[List[BarData]]
"""
if self.symbols is None:
return None
symbol = req.symbol
exchange = req.exchange
interval = req.interval
start = req.start.strftime(TS_DATE_FORMATE)
end = req.end.strftime(TS_DATE_FORMATE)
if interval is not Interval.DAILY:
return None
if exchange not in [Exchange.SSE, Exchange.SZSE]:
return None
tscode = to_ts_symbol(symbol, exchange)
# 修改查询数据逻辑，在每次5000条数据的限制下，很可能一次无法读取完
cnt = 0
df: pd.DataFrame = None
while datetime.strptime(start, TS_DATE_FORMATE) <= datetime.strptime(
end, TS_DATE_FORMATE):
# 保证每次查询最多5000天数据
start_date = datetime.strptime(start, TS_DATE_FORMATE)
simulate_end_date = min(
datetime.strptime(end, TS_DATE_FORMATE),
start_date + timedelta(days=MAX_QUERY_SIZE))
simulate_end = simulate_end_date.strftime(TS_DATE_FORMATE)
# 保证每次调用时间在60/500=0.12秒内，以保证每分钟调用次数少于500次
# begin_time = time.time()
tushare_df = None
while True:
try:
tushare_df = self.pro.query('daily',
ts_code=tscode,
start_date=start,
end_date=simulate_end)
except (requests.exceptions.SSLError,
requests.exceptions.ConnectionError) as e:
log.error(e)
# traceback.print_exc()
# ('Connection aborted.', ConnectionResetError(10054, '远程主机强迫关闭了一个现有的连接。', None, 10054, None))
if '10054' in str(e):
sleep_time = 60.0
log.info("请求过于频繁，sleep：" + str(sleep_time) + "s")
time.sleep(sleep_time)
log.info("继续发送请求：" + tscode)
continue # 继续发请求
else:
raise Exception(e) # 其他异常，抛出来
break
if tushare_df is not None:
if df is None:
df = tushare_df
else:
df = pd.concat([df, tushare_df], ignore_index=True)
# end_time = time.time()
# delta = round(end_time - begin_time, 3)
# if delta < 60 / MAX_QUERY_TIMES:
sleep_time = 0.50
log.info("sleep：" + str(sleep_time) + "s")
time.sleep(sleep_time)
cnt += 1
start = (simulate_end_date +
timedelta(days=1)).strftime(TS_DATE_FORMATE)
data: List[BarData] = []
if df is not None:
for ix, row in df.iterrows():
date = datetime.strptime(row.trade_date, '%Y%m%d')
date = CHINA_TZ.localize(date)
if pd.isnull(row['open']):
log.info(symbol + '.' + EXCHANGE_VT2TS[exchange] +
row['trade_date'] + "open_price为None")
elif pd.isnull(row['high']):
log.info(symbol + '.' + EXCHANGE_VT2TS[exchange] +
row['trade_date'] + "high_price为None")
elif pd.isnull(row['low']):
log.info(symbol + '.' + EXCHANGE_VT2TS[exchange] +
row['trade_date'] + "low_price为None")
elif pd.isnull(row['close']):
log.info(symbol + '.' + EXCHANGE_VT2TS[exchange] +
row['trade_date'] + "close_price为None")
elif pd.isnull(row['amount']):
log.info(symbol + '.' + EXCHANGE_VT2TS[exchange] +
row['trade_date'] + "volume为None")
row.fillna(0)
bar = BarData(symbol=symbol,
exchange=exchange,
interval=interval,
datetime=date,
open_price=row['open'],
high_price=row['high'],
low_price=row['low'],
close_price=row['close'],
volume=row['amount'],
gateway_name='tushare')
data.append(bar)
return data
def gen_chs():
"""1. 简体"""
NEED_CLEAR = True
log.info(os.getcwd())
if NEED_CLEAR:
print('### 正在清理...')
log.debug('正在清理...')
# 清理输出目录
dirs = ('../../输出/生成简体插件/', )
for dir in dirs:
if os.path.exists(dir):
log.info('clear %s' % dir)
shutil.rmtree(dir)
# 清理翻译中间目录
log.info('clear csv and xlsx')
for root, dirs, files in os.walk('../../translation/lang/translated/'):
for f in files:
if f.endswith('.csv') or f.endswith('.xlsx') or f.endswith(
'.lang'):
filename = os.path.join(root, f)
log.debug('remove %s' % filename)
os.remove(filename)
break
for root, dirs, files in os.walk('../../translation/'):
for f in files:
if f.endswith('.csv') or f.endswith('.xlsx'):
filename = os.path.join(root, f)
log.debug('remove %s' % filename)
os.remove(filename)
break
files = (
'../../translation/zh_translate.txt',
'../../translation/zh_translate.xlsx',
)
for f in files:
if os.path.exists(f):
log.debug('remove %s' % f)
os.remove(f)
print('### 创建目录...')
log.debug('创建目录...')
dirs = ('../../输出/生成简体插件/', )
for dir in dirs:
if not os.path.isdir(dir):
log.info('create %s' % dir)
os.makedirs(dir)
print('### 拷贝文件...')
log.debug('拷贝文件...')
for root, dirs, files in os.walk('汉化xlsx/'):
for f in files:
if f.endswith('.xlsx') and not f.startswith('~'):
filename = os.path.join(root, f)
data = load_xls(filename)
if len(data[1]) == 8: # ui 汉化文件
dst = '../../translation/'
ui_xls_file = f
log.info('copy %s to %s' % (filename, dst))
shutil.copy(filename, dst)
dst = '../../translation/lang/translated/'
log.info('copy %s to %s' % (filename, dst))
shutil.copy(filename, dst)
os.chdir('../../scripts/')
log.info(os.getcwd())
print('### 转换UI文本...')
log.debug('转换UI文本...')
execute('python export_uixls_to_txt.py ../translation/%s' % ui_xls_file)
execute('python convert_txt_to_str.py -m translation')
print('### 转换其他文本...')
log.debug('转换其他文本...')
execute('python export_langxls_to_csv.py')
os.chdir('../translation/lang/translated/')
log.info(os.getcwd())
print('### 正在编码...')
log.debug('正在编码...')
execute('EsoExtractData.exe -x zh.lang.csv')
print('### 正在校验...')
log.debug('正在校验...')
shutil.copy('zh.lang', 'zh1.lang')
execute('EsoExtractData -l zh1.lang')
num1 = get_linenum('../en.lang.csv')
num2 = get_linenum('zh.lang.csv')
num3 = get_linenum('zh1.lang.csv')
log.info('validate line num: %d, %d, %d' % (num1, num2, num3))
if not num1 == num2 == num3:
log.error('校验失败')
sys.exit(-1)
os.chdir('../../../')
log.info(os.getcwd())
print('### 正在打包...')
log.debug('正在打包...')
log.info('copy AddOns')
shutil.copytree('AddOns/', '输出/生成简体插件/AddOns')
log.info('copy lang')
shutil.copy('translation/lang/translated/zh.lang',
'输出/生成简体插件/AddOns/gamedata/lang/')
log.info('copy readme')
shutil.copy('工具/生成插件/README_chs.txt', '输出/生成简体插件/README.txt')
log.info('clear AddOns')
os.remove('输出/生成简体插件/AddOns/EsoUI/lang/.gitignore')
os.remove('输出/生成简体插件/AddOns/EsoZH/fonts/README.md')
os.remove('输出/生成简体插件/AddOns/gamedata/lang/.gitignore')
os.chdir('输出/生成简体插件/')
log.info(os.getcwd())
with open('README.txt', 'rt', encoding='utf-8') as fp:
desc = fp.read()
zip_name = 'ESO汉化插件.zip'
zipf = zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED)
zipf.comment = bytes(desc, encoding='gbk')
for root, dirs, files in os.walk('.'):
for f in files:
if f != zip_name:
zipf.write(os.path.join(root, f))
zipf.close()
def getImages(self, base_url, gid, token, imageCount, flag):
headers = getRandomHead()
pageCount = int(imageCount / 40)
if imageCount % 40 != 0:
pageCount += 1
imageUrls = []
beginPage = 0
tsequence = 0
sequence = 0
if flag:
sequence = self.db.getLastImageSequenceByGidAndToken(gid,
token) + 1
tsequence = self.db.getLastThumbImageSequenceByGidAndToken(
gid, token) + 1
beginPage = int(sequence / 40)
if sequence % 40 != 0:
beginPage += 1
for i in range(beginPage, pageCount):
info(
"info", "excrawler.getImages", "开始爬取本子详情页第" + str(i + 1) +
"页,gid:" + str(gid) + " token:" + token)
if i != 0:
url = base_url + "?p=" + str(i)
else:
url = base_url
errorCount = 0
while True:
if errorCount >= 5:
#记录出现异常的本子
self.db.insertError(gid, token)
error("异常", "excrawler.getImages",
"ParseError:连续5次未能从本子详情页获取缩略图列表")
#raise ParseError("连续5次未能从本子详情页获取缩略图列表")
try:
res = invokeRequest("获取本子详情页第" + str(i + 1) + "页",
"excrawler.getImages",
requests.get,
url,
cookies=COOKIE,
headers=headers,
timeout=30,
proxies=self.pool.getProxysequence())
except Networkerror as e:
self.db.insertError(gid, token)
error("异常", "excrawler.getImages",
"Networkerror:" + e.message)
soup = BeautifulSoup(res.text, "html.parser")
tags = soup.select("#gdt > .gdtm > div")
if len(tags) == 0:
info("信息", "excrawler.getImages",
"本子详情页缩略图列表中没有图片,url:" + url + "html:" + res.text)
errorCount += 1
continue
images = []
for tag in tags:
timageUrl = re.findall(r"url\((.+)\)", tag['style'])[0]
a = tag.select("a")[0]
href = a['href']
img = {
'gid': gid,
'token': token,
'sequence': sequence,
'url': href
}
images.append(img)
sequence += 1
if timageUrl not in imageUrls:
timg = {
'gid': gid,
'token': token,
'sequence': tsequence,
'url': timageUrl
}
self.db.insertThumbimage(timg)
tsequence += 1
imageUrls.append(timageUrl)
self.db.insertEroimage(images)
break
def gen_chs_force():
"""3. 强制简体，用原版做一次繁简转换"""
NEED_CLEAR = True
log.info(os.getcwd())
if NEED_CLEAR:
print('### 正在清理...')
log.debug('正在清理...')
# 清理输出目录
dirs = ('../../输出/生成简体插件/', )
for dir in dirs:
if os.path.exists(dir):
log.info('clear %s' % dir)
shutil.rmtree(dir)
# 清理翻译中间目录
files = (
'../../translation/STOthers_ts.txt',
'../../translation/lang/translated/zh1.lang',
'../../translation/lang/translated/zh1.lang.csv',
)
for f in files:
if os.path.exists(f):
log.debug('remove %s' % f)
os.remove(f)
for root, dirs, files in os.walk('../../translation/'):
for f in files:
if f.endswith('.xlsx'):
filename = os.path.join(root, f)
log.debug('remove %s' % filename)
os.remove(filename)
break
print('### 创建目录...')
log.debug('创建目录...')
dirs = ('../../输出/生成简体插件/', )
for dir in dirs:
if not os.path.isdir(dir):
log.info('create %s' % dir)
os.makedirs(dir)
print('### 分析对照表...')
log.debug('分析对照表...')
for root, dirs, files in os.walk('繁简对照'):
for f in files:
if f.endswith('.xlsx') and not f.startswith('~'):
filename = os.path.join(root, f)
dst = '../../translation/'
log.info('copy %s to %s', filename, dst)
shutil.copy(filename, dst)
cht_to_chs_file = f
break
os.chdir('../../translation/')
log.info(os.getcwd())
execute('python ../scripts/xls2csv.py "%s" STOthers_ts.txt' %
cht_to_chs_file)
with open('STOthers_ts.txt', 'rt', encoding='utf-8') as fp:
lines = fp.readlines()
with open('STOthers_ts.txt', 'wt', encoding='utf-8') as fp:
fp.write(''.join(lines[1:]))
os.chdir('../scripts/')
log.info(os.getcwd())
print('### 繁简转换...')
log.debug('繁简转换...')
# 直接覆盖吧
src_dst = (
(
'../AddOns/EsoUI/lang/zh_pregame.str',
'../AddOns/EsoUI/lang/zh_pregame.str',
),
(
'../AddOns/EsoUI/lang/zh_client.str',
'../AddOns/EsoUI/lang/zh_client.str',
),
(
'../translation/lang/translated/zh.lang.csv',
'../translation/lang/translated/zh.lang.csv',
),
)
for src, dst in src_dst:
execute('python convert_to_chs.py %s %s' % (src, dst))
os.chdir('../translation/lang/translated/')
log.info(os.getcwd())
print('### 正在编码...')
log.debug('正在编码...')
execute('EsoExtractData.exe -x zh.lang.csv')
print('### 正在校验...')
log.debug('正在校验...')
shutil.copy('zh.lang', 'zh1.lang')
execute('EsoExtractData -l zh1.lang')
num1 = get_linenum('../en.lang.csv')
num2 = get_linenum('zh.lang.csv')
num3 = get_linenum('zh1.lang.csv')
log.info('validate line num: %d, %d, %d' % (num1, num2, num3))
if not num1 == num2 == num3:
log.error('校验失败')
sys.exit(-1)
os.chdir('../../../')
log.info(os.getcwd())
print('### 正在打包...')
log.debug('正在打包...')
log.info('copy AddOns')
shutil.copytree('AddOns/', '输出/生成简体插件/AddOns')
log.info('copy lang')
shutil.copy('translation/lang/translated/zh.lang',
'输出/生成简体插件/AddOns/gamedata/lang/')
log.info('copy readme')
shutil.copy('工具/生成插件/README_chs.txt', '输出/生成简体插件/README.txt')
log.info('clear AddOns')
os.remove('输出/生成简体插件/AddOns/EsoUI/lang/.gitignore')
os.remove('输出/生成简体插件/AddOns/EsoZH/fonts/README.md')
os.remove('输出/生成简体插件/AddOns/gamedata/lang/.gitignore')
os.chdir('输出/生成简体插件/')
log.info(os.getcwd())
with open('README.txt', 'rt', encoding='utf-8') as fp:
desc = fp.read()
zip_name = 'ESO汉化插件.zip'
zipf = zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED)
zipf.comment = bytes(desc, encoding='gbk')
for root, dirs, files in os.walk('.'):
for f in files:
if f != zip_name:
zipf.write(os.path.join(root, f))
zipf.close()
text = gzip.GzipFile(fileobj=cStringIO.StringIO(text)).read()
except httplib.IncompleteRead as e:
text = e.partial
except (Exception), e:
logging.user(
self.request,
"~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % e)
logging.error('error fetch_request'+str(e)+\
# ' feed_id:'+str(self.story.story_feed_id)+\
' stroy_link:'+str(self.story.story_permalink))
return
finally:
opener.close()
if not text:
logging.error('error fetch text: text is null')
return
#soup = BeautifulSoup(text)
#text = soup.renderContents()
try:
original_text_doc = readability.Document(
text, url=self.story.story_permalink)
content = original_text_doc.summary(html_partial=True)
print "the length of content: %s" % len(content)
#content = content.encode("utf-8")
except readability.Unparseable, e:
logging.error('error getting summary: '+str(e)+\
# ' feed_id:'+str(self.story.story_feed_id)+\
' stroy_link:'+str(self.story.story_permalink))
# if settings.SEND_ERROR_MAILS:
# mail_admins("Error in text_importer Build Document",str(e)+\
def main():
args = parse_args()
# --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------
log.info('Creating OpenVINO Runtime Core')
core = Core()
# --------------------------- Step 2. Read a model --------------------------------------------------------------------
if args.model:
log.info(f'Reading the model: {args.model}')
# (.xml and .bin files) or (.onnx file)
model = core.read_model(args.model)
# --------------------------- Step 3. Apply preprocessing -------------------------------------------------------------
if args.output_layers:
output_layer_names, output_layer_ports = parse_outputs_from_args(
args)
model.add_outputs(list(zip(output_layer_names,
output_layer_ports)))
if args.layout:
layouts = parse_input_layouts(args, model.inputs)
ppp = PrePostProcessor(model)
for i in range(len(model.inputs)):
ppp.input(i).tensor().set_element_type(Type.f32)
input_name = model.input(i).get_any_name()
if args.layout and input_name in layouts.keys():
ppp.input(i).tensor().set_layout(Layout(layouts[input_name]))
ppp.input(i).model().set_layout(Layout(layouts[input_name]))
for i in range(len(model.outputs)):
ppp.output(i).tensor().set_element_type(Type.f32)
model = ppp.build()
if args.batch_size:
batch_size = args.batch_size if args.context_window_left == args.context_window_right == 0 else 1
if any([not _input.node.layout.empty for _input in model.inputs]):
set_batch(model, batch_size)
else:
log.warning(
'Layout is not set for any input, so custom batch size is not set'
)
# ---------------------------Step 4. Configure plugin ---------------------------------------------------------
devices = args.device.replace('HETERO:', '').split(',')
plugin_config = {}
if 'GNA' in args.device:
gna_device_mode = devices[0] if '_' in devices[0] else 'GNA_AUTO'
devices[0] = 'GNA'
plugin_config['GNA_DEVICE_MODE'] = gna_device_mode
plugin_config['GNA_PRECISION'] = f'I{args.quantization_bits}'
plugin_config['GNA_EXEC_TARGET'] = args.exec_target
plugin_config['GNA_PWL_MAX_ERROR_PERCENT'] = str(args.pwl_me)
# Set a GNA scale factor
if args.import_gna_model:
if args.scale_factor:
log.warning(
f'Custom scale factor will be used for imported GNA model: {args.import_gna_model}'
)
set_scale_factors(plugin_config, parse_scale_factors(args))
else:
log.info(
f'Using scale factor from the imported GNA model: {args.import_gna_model}'
)
else:
if args.scale_factor:
set_scale_factors(plugin_config, parse_scale_factors(args))
else:
scale_factors = []
for file_name in re.split(', |,', args.input):
_, utterances = read_utterance_file(file_name)
scale_factors.append(get_scale_factor(utterances[0]))
log.info(
'Using scale factor(s) calculated from first utterance')
set_scale_factors(plugin_config, scale_factors)
if args.export_embedded_gna_model:
plugin_config[
'GNA_FIRMWARE_MODEL_IMAGE'] = args.export_embedded_gna_model
plugin_config[
'GNA_FIRMWARE_MODEL_IMAGE_GENERATION'] = args.embedded_gna_configuration
if args.performance_counter:
plugin_config['PERF_COUNT'] = 'YES'
device_str = f'HETERO:{",".join(devices)}' if 'HETERO' in args.device else devices[
0]
# --------------------------- Step 5. Loading model to the device -----------------------------------------------------
log.info('Loading the model to the plugin')
if args.model:
compiled_model = core.compile_model(model, device_str, plugin_config)
else:
with open(args.import_gna_model, 'rb') as f:
buf = BytesIO(f.read())
compiled_model = core.import_model(buf, device_str, plugin_config)
# --------------------------- Exporting GNA model using InferenceEngine AOT API ---------------------------------------
if args.export_gna_model:
log.info(f'Writing GNA Model to {args.export_gna_model}')
user_stream = compiled_model.export_model()
with open(args.export_gna_model, 'wb') as f:
f.write(user_stream)
return 0
if args.export_embedded_gna_model:
log.info(
f'Exported GNA embedded model to file {args.export_embedded_gna_model}'
)
log.info(
f'GNA embedded model export done for GNA generation {args.embedded_gna_configuration}'
)
return 0
# --------------------------- Step 6. Set up input --------------------------------------------------------------------
if args.input_layers:
input_layer_names = re.split(', |,', args.input_layers)
else:
input_layer_names = [
_input.any_name for _input in compiled_model.inputs
]
input_file_names = re.split(', |,', args.input)
if len(input_layer_names) != len(input_file_names):
log.error(
f'Number of model inputs ({len(compiled_model.inputs)}) is not equal '
f'to number of ark files ({len(input_file_names)})')
sys.exit(-3)
input_file_data = [
read_utterance_file(file_name) for file_name in input_file_names
]
infer_data = [{
input_layer_names[j]: input_file_data[j].utterances[i]
for j in range(len(input_layer_names))
} for i in range(len(input_file_data[0].utterances))]
if args.output_layers:
output_layer_names, output_layer_ports = parse_outputs_from_args(args)
# If a name of output layer contains a port number then concatenate output_layer_names and output_layer_ports
if ':' in compiled_model.outputs[0].any_name:
output_layer_names = [
f'{output_layer_names[i]}:{output_layer_ports[i]}'
for i in range(len(output_layer_names))
]
else:
output_layer_names = [compiled_model.outputs[0].any_name]
if args.output:
output_file_names = re.split(', |,', args.output)
if len(output_layer_names) != len(output_file_names):
log.error(
'The number of output files is not equal to the number of model outputs.'
)
sys.exit(-6)
if args.reference:
reference_file_names = re.split(', |,', args.reference)
if len(output_layer_names) != len(reference_file_names):
log.error(
'The number of reference files is not equal to the number of model outputs.'
)
sys.exit(-5)
reference_file_data = [
read_utterance_file(file_name)
for file_name in reference_file_names
]
references = [{
output_layer_names[j]: reference_file_data[j].utterances[i]
for j in range(len(output_layer_names))
} for i in range(len(input_file_data[0].utterances))]
# --------------------------- Step 7. Create infer request ------------------------------------------------------------
infer_request = compiled_model.create_infer_request()
# --------------------------- Step 8. Do inference --------------------------------------------------------------------
log.info('Starting inference in synchronous mode')
results = []
total_infer_time = 0
for i in range(len(infer_data)):
start_infer_time = default_timer()
# Reset states between utterance inferences to remove a memory impact
for state in infer_request.query_state():
state.reset()
results.append(
do_inference(
infer_data[i],
infer_request,
args.context_window_left,
args.context_window_right,
))
infer_time = default_timer() - start_infer_time
total_infer_time += infer_time
num_of_frames = infer_data[i][input_layer_names[0]].shape[0]
avg_infer_time_per_frame = infer_time / num_of_frames
# --------------------------- Step 9. Process output ------------------------------------------------------------------
log.info('')
log.info(f'Utterance {i}:')
log.info(f'Total time in Infer (HW and SW): {infer_time * 1000:.2f}ms')
log.info(f'Frames in utterance: {num_of_frames}')
log.info(
f'Average Infer time per frame: {avg_infer_time_per_frame * 1000:.2f}ms'
)
for name in output_layer_names:
log.info('')
log.info(f'Output blob name: {name}')
log.info(f'Number scores per frame: {results[i][name].shape[1]}')
if args.reference:
log.info('')
compare_with_reference(results[i][name], references[i][name])
if args.performance_counter:
if 'GNA' in args.device:
total_cycles = infer_request.profiling_info[
0].real_time.total_seconds()
stall_cycles = infer_request.profiling_info[
1].real_time.total_seconds()
active_cycles = total_cycles - stall_cycles
frequency = 10**6
if args.arch == 'CORE':
frequency *= GNA_CORE_FREQUENCY
else:
frequency *= GNA_ATOM_FREQUENCY
total_inference_time = total_cycles / frequency
active_time = active_cycles / frequency
stall_time = stall_cycles / frequency
log.info('')
log.info('Performance Statistics of GNA Hardware')
log.info(
f' Total Inference Time: {(total_inference_time * 1000):.4f} ms'
)
log.info(f' Active Time: {(active_time * 1000):.4f} ms')
log.info(f' Stall Time: {(stall_time * 1000):.4f} ms')
log.info('')
log.info(f'Total sample time: {total_infer_time * 1000:.2f}ms')
if args.output:
for i, name in enumerate(output_layer_names):
data = [
results[i][name]
for i in range(len(input_file_data[0].utterances))
]
write_utterance_file(output_file_names[i], input_file_data[0].keys,
data)
log.info(f'File {output_file_names[i]} was created!')
# ----------------------------------------------------------------------------------------------------------------------
log.info(
'This sample is an API example, '
'for any performance measurements please use the dedicated benchmark_app tool\n'
)
return 0
log.warning('(x) Exit')
arg = input()[0]
print()
model = ContentBasedRecommender()
if arg == 't':
# recs = model.recommend_batch(userids=data.get_target_playlists(), urm=data.get_urm_train())
# model.evaluate(recommendations=recs, test_urm=data.get_urm_test())
model.test(distance=model.SIM_SPLUS, k=500,alpha=0.75,beta=1,shrink=500,l=0.5,c=0.5)
elif arg == 'r':
log.info('Wanna save for evaluation (y/n)?')
choice = input()[0] == 'y'
model.fit(urm=data.get_urm_train_2(),icm=data.get_icm(), distance=model.SIM_SPLUS,k=500,shrink=500,alpha=0.75,beta=1,l=0.5,c=0.5)
print('Saving the R^...')
model.save_r_hat(evaluation=choice)
elif arg == 's':
model.fit(urm=data.get_urm_train_2(),icm=data.get_icm(), distance=model.SIM_SPLUS,k=500,shrink=500,alpha=0.75,beta=1,l=0.5,c=0.5)
print('Saving the similarity matrix...')
sps.save_npz('raw_data/saved_sim_matrix_evaluation_2/{}'.format(model.name), model.get_sim_matrix())
# elif arg == 'v':
# model.validate(....)
elif arg == 'x':
pass
else:
log.error('Wrong option!')
# recs = model.recommend_batch(userids=data.get_target_playlists(), urm=data.get_urm_train())
# recs_seq = model.recommend_batch(userids=data.get_sequential_target_playlists(), urm=data.get_urm_train())
# model.evaluate(recommendations=recs, test_urm=data.get_urm_test())
# model.evaluate(recommendations=recs_seq, test_urm=data.get_urm_test())
def export_xls(raw):
"""
Выгрузка структуры телефонной книги в excel
:param raw: {string: {string: [[string]]}}, структура тел. книги {организация: {отдел: [[данные_сотрудника]]}}
:return: bool, True - если тел. книга выгружена, False - если произошла ошибка выгрузки
"""
wb = xlwt.Workbook()
ws = wb.add_sheet('Телефонная книга')
# Заголовок
ws.write_merge(0, 0, 0, 1, '', ts.sh0)
ws.write(0, 2, 'ФИО', ts.sh1)
ws.write(0, 3, 'Должность', ts.sh2)
ws.write(0, 4, 'Вн. тел.', ts.sh2)
ws.write(0, 5, 'Гор. тел.', ts.sh2)
ws.write(0, 6, 'Почта', ts.sh3)
line = 1
item_len = [0, 0, 0, 0, 0, 0] # Для подсчёта ширины колонок
item_style = [ts.ss1, ts.ss2, ts.ss2, ts.ss2, ts.ss3] # Стили для полей сотрудников
for kc in sorted(raw):
# Наименование организации
ws.write_merge(line, line, 0, 6, kc, ts.so0)
ws.row(line).level = 0
line += 1
ws.row(line).level = 1
for kd in sorted(raw[kc]):
# Наименование отдела
ws.write(line, 0, '', ts.sd0)
ws.write_merge(line, line, 1, 6, kd, ts.sd1)
ws.row(line).level = 1
line += 1
ws.row(line).level = 2
for item in sorted(raw[kc][kd]):
ws.write_merge(line, line, 0, 1, '', ts.ss0)
for k, field in enumerate(item):
# Поля сотрудника: ФИО, Должность, Вн. тел., Гор. тел., Почта
ws.write(line, k + 2, field, item_style[k])
# Считаем максимальную ширину для каждой колонки
item_len[k] = max(item_len[k], len(field))
line += 1
ws.row(line).level = 2
# Подчёркиваем таблицу
ws.write_merge(line, line, 0, 6, '', ts.sb0)
ws.row(line).level = 0
# Задаем ширину колонок, для организации и отдела ширина фиксирована
ws.col(0).width = int(36.5 * 17)
ws.col(1).width = int(36.5 * 17)
# Для остальных колонок ширина вычисляется в зависимости от длинны полей
for k, v in enumerate(item_len):
ws.col(k + 2).width = int(36.5 * 7.3 * v)
# Закрепляем заголовок
ws.panes_frozen = True
ws.horz_split_pos = 1
path = get_options('main', 'xls_path', True)
if not path:
log.critical('Ошибка чтения конфигурационного файла, см. ошибки выше')
return
try:
wb.save(path)
except PermissionError as e:
log.error('Недостаточно прав для сохранения файла: %s' % e.filename)
return
except FileNotFoundError as e:
log.error('Неверный путь или имя файла: %s' % e.filename)
return
return True
feed.save_feed_history(e.code, e.msg, e.fp.read())
fetched_feed = None
except Feed.DoesNotExist, e:
logging.debug(' ---> [%-30s] ~FRFeed is now gone...' %
(unicode(feed_id)[:30]))
continue
except TimeoutError, e:
logging.debug(' ---> [%-30s] ~FRFeed fetch timed out...' %
(feed.title[:30]))
feed.save_feed_history(505, 'Timeout', '')
feed_code = 505
fetched_feed = None
except Exception, e:
logging.debug('[%d] ! -------------------------' % (feed_id, ))
tb = traceback.format_exc()
logging.error(tb)
logging.debug('[%d] ! -------------------------' % (feed_id, ))
ret_feed = FEED_ERREXC
feed = Feed.get_by_id(getattr(feed, 'pk', feed_id))
feed.save_feed_history(500, "Error", tb)
feed_code = 500
fetched_feed = None
mail_feed_error_to_admin(feed, e, local_vars=locals())
if (not settings.DEBUG and hasattr(settings, 'RAVEN_CLIENT')
and settings.RAVEN_CLIENT):
settings.RAVEN_CLIENT.captureException()
if not feed_code:
if ret_feed == FEED_OK:
feed_code = 200
elif ret_feed == FEED_SAME:
i += len(w)
if i + len(w) <= 18:
i += len(w)
final_sentence += ' ' + w
else:
i = len(w)
final_sentence += '\n ' + w
return final_sentence
# -----------------------------------
# I/O Procedures
# -----------------------------------
def read_file(path):
with open(path, "r") as f:
sentences = f.readlines()
return sentences
def save_file(path, image, name_file):
log.info(f'4/4 Saving image: {name_file}')
image.save(path + '/' + name_file)
if __name__ == '__main__':
if len(sys.argv) < 3:
log.error('Bad arguments. Required: Images text file path')
run(sys.argv[1], sys.argv[2])
def main():
lang = 'zh'
mode = 'both' # origin, translation, both
# getopt
try:
opts, args = getopt.getopt(sys.argv[1:], 'l:m:h')
except getopt.GetoptError as e:
log.error(e)
usage()
sys.exit(2)
for o, a in opts:
if o == '-l':
lang = a.lower()
elif o == '-m':
mode = a.lower()
if mode not in ('origin', 'translation', 'both'):
usage()
sys.exit(2)
elif o == '-h':
usage()
return
cd = sys.path[0]
translation_path = os.path.join(cd, '../translation')
dest_path = os.path.join(cd, '../AddOns/EsoUI/lang')
# load header
header_file = os.path.join(translation_path, 'str_header.txt')
with open(header_file, 'rt', encoding='utf-8') as fp:
header = fp.readlines()
ui_mgr_pregame = UiMgr()
ui_mgr_client = UiMgr()
# load lua
pregame_src = os.path.join(translation_path, 'en_pregame.lua')
ui_mgr_pregame.load_lua_file(pregame_src)
client_src = os.path.join(translation_path, 'en_client.lua')
ui_mgr_client.load_lua_file(client_src)
# load translation
translate_file = os.path.join(translation_path, '%s_translate.txt' % lang)
with open(translate_file, 'rt', encoding='utf-8') as fp:
lines = fp.readlines()
ui_mgr_pregame.apply_translate_from_txt_lines(lines)
ui_mgr_client.apply_translate_from_txt_lines(lines)
log.info('mode: %s' % mode)
# save str
pregame_dest = os.path.join(dest_path, '%s_pregame.str' % lang)
pregame_lines = ui_mgr_pregame.get_str_lines(mode)
log.info('save to %s.' % pregame_dest)
with open(pregame_dest, 'wt', encoding='utf-8') as fp:
fp.writelines(header)
fp.writelines(pregame_lines)
client_dest = os.path.join(dest_path, '%s_client.str' % lang)
client_lines = ui_mgr_client.get_str_lines(mode)
log.info('save to %s.' % client_dest)
with open(client_dest, 'wt', encoding='utf-8') as fp:
fp.writelines(header)
fp.writelines(client_lines)
# save en str (fix fonts)
if lang != 'en':
pregame_dest = os.path.join(dest_path, 'en_pregame.str')
pregame_lines = ui_mgr_pregame.get_str_lines(mode)
log.info('save to %s.' % pregame_dest)
with open(pregame_dest, 'wt', encoding='utf-8') as fp:
fp.writelines(header)
# keep version info
for line in pregame_lines:
if line.startswith('[SI_VERSION] ='):
fp.write(line + '\n')
client_dest = os.path.join(dest_path, 'en_client.str')
client_lines = ui_mgr_client.get_str_lines(mode)
log.info('save to %s.' % client_dest)
with open(client_dest, 'wt', encoding='utf-8') as fp:
fp.writelines(header)
# keep version info
for line in client_lines:
if line.startswith('[SI_VERSION] ='):
fp.write(line + '\n')
else:
log.error('WRONG FOLDER')
def wizard_CF():
print('Wizard soon...')
def wizard_CB():
print('Wizard soon...')
def wizard_misc():
print('Wizard soon...')
if __name__ == '__main__':
print()
log.error(
'(¯`·.¸¸.·´¯`·.¸¸.-> GabDamPar ® - The AIO Recommender System <-.¸¸.·´¯`·.¸¸.·´¯)'
)
print()
menu.show('Which model do you want to run?', {
'1': ('Collaborative Filtering', wizard_CF),
'2': ('Content Based', wizard_CB),
'3': ('Miscellaneous', wizard_misc),
'4': ('Hybrid', wizard_hybrid),
},
main_menu=True)
