def __init__(self, parent=None, htmlFile='', txtFile=''):
super(FileViewTab, self).__init__(parent)
self.main = parent
self.waiting = True
if htmlFile != '':
# un QWebView pour afficher si c'est un fichier html :
self.bigTruc = QtWebKit.QWebView(self)
self.bigTruc.page().setLinkDelegationPolicy(
QtWebKit.QWebPage.DelegateAllLinks)
self.bigTruc.linkClicked.connect(self.link_clicked)
cssFile = utils.u('{0}/{1}/{2}').format(parent.beginDir, 'libs',
'base.css')
url = QtCore.QUrl().fromLocalFile(cssFile)
self.bigTruc.settings().setUserStyleSheetUrl(url)
else:
# ou un QTextEdit pour un fichier texte :
self.bigTruc = QtGui.QTextEdit()
self.bigTruc.setReadOnly(True)
mainLayout = QtGui.QHBoxLayout()
mainLayout.addWidget(self.bigTruc)
self.setLayout(mainLayout)
# on ouvre le fichier :
if htmlFile != '':
htmlFile = utils.u('{0}/{1}').format(parent.beginDir, htmlFile)
url = QtCore.QUrl().fromLocalFile(htmlFile)
self.bigTruc.load(url)
else:
inFile = QtCore.QFile(txtFile)
if inFile.open(QtCore.QFile.ReadOnly | QtCore.QFile.Text):
stream = QtCore.QTextStream(inFile)
stream.setCodec('UTF-8')
self.bigTruc.setPlainText(stream.readAll())
inFile.close()
self.waiting = False
def __init__(self, parent=None, htmlFile='', txtFile=''):
super(FileViewTab, self).__init__(parent)
self.main = parent
self.waiting = True
if htmlFile != '':
# un QWebView pour afficher si c'est un fichier html :
self.bigTruc = QtWebKit.QWebView(self)
self.bigTruc.page().setLinkDelegationPolicy(QtWebKit.QWebPage.DelegateAllLinks)
self.bigTruc.linkClicked.connect(self.link_clicked)
cssFile = utils.u('{0}/{1}/{2}').format(parent.beginDir, 'libs', 'base.css')
url = QtCore.QUrl().fromLocalFile(cssFile)
self.bigTruc.settings().setUserStyleSheetUrl(url)
else:
# ou un QTextEdit pour un fichier texte :
self.bigTruc = QtGui.QTextEdit()
self.bigTruc.setReadOnly(True)
mainLayout = QtGui.QHBoxLayout()
mainLayout.addWidget(self.bigTruc)
self.setLayout(mainLayout)
# on ouvre le fichier :
if htmlFile != '':
htmlFile = utils.u('{0}/{1}').format(parent.beginDir, htmlFile)
url = QtCore.QUrl().fromLocalFile(htmlFile)
self.bigTruc.load(url)
else:
inFile = QtCore.QFile(txtFile)
if inFile.open(QtCore.QFile.ReadOnly | QtCore.QFile.Text):
stream = QtCore.QTextStream(inFile)
stream.setCodec('UTF-8')
self.bigTruc.setPlainText(stream.readAll())
inFile.close()
self.waiting = False
def parse_mesh(data, mesh_id):
position = 0
shader, position = utils.parse_string_nul(data, position)
texture, position = utils.parse_string_nul(data, position)
mesh_info, position = u('IffII', data, position)
flags, min_scale, max_scale, vertices_count, indices_count = mesh_info
vertices = []
uvs = []
for vertex_id in range(vertices_count):
vertex_data, position = u('5f', data, position)
position_x, position_y, position_z, uv_x, uv_y = vertex_data
vertices.append((position_x, position_z, position_y))
uvs.append((uv_x, 1 - uv_y))
triangles = []
for index_id in range(indices_count // 3):
triangle_data, position = u('3H', data, position)
index_1, index_2, index_3 = triangle_data
triangles.append((index_1, index_3, index_2))
import_utils.crete_mesh(vertices,
triangles,
uvs=uvs,
image='details\\build_details')
def __init__(self, parent=None):
super(ScreenPage, self).__init__(parent)
self.main = parent.main
# configuration de la variable globale SCREEN_MODE :
groupBoxScreenMode = QtGui.QGroupBox('')
title = QtGui.QApplication.translate('main', 'ScreenMode')
titleLabel = QtGui.QLabel(utils.u('<p align="center"><b>{0}</b></p>').format(title))
radio1 = QtGui.QRadioButton(QtGui.QApplication.translate('main', 'FullSpace'))
self.radioScreenMode = QtGui.QRadioButton(QtGui.QApplication.translate('main', 'FullScreen'))
if utils.SCREEN_MODE == 'FULL_SCREEN':
self.radioScreenMode.setChecked(True)
else:
radio1.setChecked(True)
vLayout = QtGui.QVBoxLayout()
vLayout.addWidget(titleLabel)
vLayout.addWidget(radio1)
vLayout.addWidget(self.radioScreenMode)
vLayout.addStretch(1)
groupBoxScreenMode.setLayout(vLayout)
bigEditorScreenMode = QtGui.QTextEdit()
bigEditorScreenMode.setReadOnly(True)
bigEditorScreenMode.setText(
QtGui.QApplication.translate('main', ''
'<p align=left><b>FullSpace: </b>'
'the application use all the free space on desktop.</p>'
'<p></p>'
'<p align=left><b>FullScreen: </b>'
'choose this if you ave problems with FullSpace mode.</p>'
'<p></p>'
'<p align=center><b>If you change this, you need to restart application.</b></p>'))
# configuration de la variable globale SCREEN_NUMBER :
groupBoxScreenNumber = QtGui.QGroupBox('')
title = QtGui.QApplication.translate('main', 'ScreenNumber')
titleLabel = QtGui.QLabel(utils.u('<p align="center"><b>{0}</b></p>').format(title))
self.comboBoxScreenNumber = QtGui.QComboBox()
total = QtGui.QApplication.desktop().screenCount()
if utils.SCREEN_NUMBER >= total:
total = utils.SCREEN_NUMBER + 1
for i in range(total):
self.comboBoxScreenNumber.addItem(utils.u('{0} {1}').format(QtGui.QApplication.translate('main', 'Screen'), i), i)
self.comboBoxScreenNumber.setCurrentIndex(utils.SCREEN_NUMBER)
vLayout = QtGui.QVBoxLayout()
vLayout.addWidget(titleLabel)
vLayout.addWidget(self.comboBoxScreenNumber)
vLayout.addStretch(1)
groupBoxScreenNumber.setLayout(vLayout)
bigEditorScreenNumber = QtGui.QTextEdit()
bigEditorScreenNumber.setReadOnly(True)
bigEditorScreenNumber.setText(
QtGui.QApplication.translate('main', '<P></P>'
'<P ALIGN=LEFT>Here you can select on which screen you will to work.</P>'))
# mise en place :
layout = QtGui.QGridLayout()
layout.addWidget(groupBoxScreenMode, 0, 0)
layout.addWidget(bigEditorScreenMode, 0, 1)
layout.addWidget(groupBoxScreenNumber, 1, 0)
layout.addWidget(bigEditorScreenNumber, 1, 1)
self.setLayout(layout)
def __init__(self, parent=None):
super(KidWindowPage, self).__init__(parent)
self.main = parent.main
# configuration de la variable iconSize :
groupBoxIconSize = QtGui.QGroupBox('')
title = QtGui.QApplication.translate('main', 'IconSize')
titleLabel = QtGui.QLabel(utils.u('<p align="center"><b>{0}</b></p>').format(title))
integerLabel = QtGui.QLabel(QtGui.QApplication.translate('main',
'Enter a value between {0} and {1}:').format(8, 128))
self.integerSpinBoxIconSize = QtGui.QSpinBox()
self.integerSpinBoxIconSize.setRange(8, 128)
self.integerSpinBoxIconSize.setSingleStep(1)
self.integerSpinBoxIconSize.setValue(self.main.iconSizeKid.width())
hLayout = QtGui.QHBoxLayout()
hLayout.addWidget(integerLabel)
hLayout.addWidget(self.integerSpinBoxIconSize)
vLayout = QtGui.QVBoxLayout()
vLayout.addWidget(titleLabel)
vLayout.addLayout(hLayout)
groupBoxIconSize.setLayout(vLayout)
# configuration des actions visibles dans les toolBars :
groupBoxVisibleActions = QtGui.QGroupBox('')
title = QtGui.QApplication.translate('main', 'VisibleActions')
titleLabel = QtGui.QLabel(utils.u('<p align="center"><b>{0}</b></p>').format(title))
self.toolbarActionsList = QtGui.QListWidget()
boldFont = QtGui.QFont()
boldFont.setBold(True)
for toolBar in self.main.toolsWindow.toolBarsList:
item = QtGui.QListWidgetItem(utils.u('{0} :').format(toolBar.windowTitle()))
item.setFont(boldFont)
self.toolbarActionsList.addItem(item)
for action in self.main.toolsWindow.toolBars[toolBar]:
if action in self.main.toolsWindow.actionsState:
actionName = self.main.toolsWindow.actionsState[action][0]
if isinstance(action, QtGui.QWidgetAction):
item = QtGui.QListWidgetItem(action.defaultWidget().itemIcon(0), actionName)
elif isinstance(action, QtGui.QAction):
item = QtGui.QListWidgetItem(action.icon(), actionName)
if self.main.toolsWindow.actionsState[action][2]:
item.setCheckState(QtCore.Qt.Checked)
else:
item.setCheckState(QtCore.Qt.Unchecked)
item.setData(QtCore.Qt.UserRole, action)
self.toolbarActionsList.addItem(item)
vLayout = QtGui.QVBoxLayout()
vLayout.addWidget(titleLabel)
vLayout.addWidget(self.toolbarActionsList)
groupBoxVisibleActions.setLayout(vLayout)
# mise en place :
layout = QtGui.QVBoxLayout()
layout.addWidget(groupBoxIconSize)
layout.addWidget(groupBoxVisibleActions)
self.setLayout(layout)
def parse_cform(s):
I, H, B, f, p, verts, faces = 'I', 'H', 'B', 'f', 0, [], []
version, p = u(I, s, p)
vertex_count, p = u(I, s, p)
face_count, p = u(I, s, p)
bbox_diagonal, p = u(6*f, s, p)
for i in range(vertex_count):
vertex, p = u(3*f, s, p)
verts.append((vertex[0], vertex[2], vertex[1]))
material_indices = []
materials = {}
for i in range(face_count):
face, p = u(3*I, s, p)
'''
material_id = face_data & 0x3fff
suppress_shadows = face_data & 0x4000
suppress_wallmark = face_data & 0x8000
sector_id = face_data >> 16 & 0xffff
'''
material_id, p = u(B, s, p)
flags, p = u(B, s, p)
sector_id, p = u(H, s, p)
faces.append((face[0], face[2], face[1]))
material_indices.append(material_id)
materials[material_id] = True
materials = list(materials.keys())
return verts, faces, material_indices, materials
def save_record(metric_name, ts, val):
global cnt
dir = prepare_dirs(metric_name)
hour_ts = math.floor(ts / 3600) * 3600
fname = os.path.join(u(dir), u(hour_ts))
fname = '{}.csv'.format(u(fname))
fhandle = get_fhandle(fname)
fhandle.write(','.join(u([ts, val])))
fhandle.write('\n')
rds.lpush('metrics_queue', ','.join(u([metric_name, ts, val])))
def update(self, people_entry):
""" 更新用户档案 """
url, image_url, blog_url = parse_urls(people_entry)
self.user_name = u(people_entry.uid.text)
self.screen_name = u(people_entry.title.text)
self.location = u(people_entry.location.text)
self.content = u(people_entry.content.text)
self.url = url
self.image_url = image_url
self.blog_url = blog_url
self.put()
def save_record(tenant, metric_name, ts, val, metric_props):
dir = prepare_dirs(metric_name)
hour_ts = math.floor(ts/3600)*3600
fname = os.path.join(u(dir), u(hour_ts))
fname = '{}.csv'.format(u(fname))
fhandle = get_fhandle(fname)
fhandle.write(','.join(u([ts, val])))
fhandle.write('\n')
#
if not metric_props:
metric_props = {}
update_db_index(tenant, metric_name, ts, val, fname, metric_props)
def insert(cls, people_entry):
""" 新建用户档案 """
url, image_url, blog_url = parse_urls(people_entry)
new_profile = DoubanProfile(douban_id=parse_id(people_entry), \
user_name=u(people_entry.uid.text), \
screen_name=u(people_entry.title.text), \
location=u(people_entry.location.text), \
content=u(people_entry.content.text), \
url=url, \
image_url=image_url, \
blog_url=blog_url)
new_profile.put()
return new_profile
def raw(self):
r = self.__dict__["data"]
if isinstance(r, list):
ret = []
for x in r:
ret += [x.raw() if isinstance(x, JSONObject) else x]
return ret
else:
ret = {}
for x in r:
ret[str(x)] = r[x].raw() if isinstance(r[x], JSONObject) else u(r[x])
if isinstance(ret[x], unicode):
ret[str(x)] = u(str(ret[x]))
return ret
def summarize_metric_files(metric_name, ts_start_ts, ts_end_ts):
mdir, flist = files_for_metrics(metric_name, ts_start_ts, ts_end_ts)
dfs = []
for f in u(flist):
if '-' in f:
continue
f = os.path.join(u(mdir), u(f))
df = pd.read_csv(f,
header=None,
parse_dates=[0],
index_col=0,
date_parser=date_parser)
print(">>>", f, df.index[0], df.index[-1], len(df))
dfs.append(df)
return dfs
def parse_slots(data):
position = 0
coord_y = []
for slot in range(len(data) // 16):
slot_data, position = u('IIHHHH', data, position)
y_base = slot_data[0] & 0x3ff
y_height = (slot_data[0] >> 12) & 0xff
id0 = (slot_data[0] >> 20) & 0x3f
id1 = (slot_data[0] >> 26) & 0x3f
id2 = (slot_data[1]) & 0x3f
id3 = (slot_data[1] >> 6) & 0x3f
c_dir = (slot_data[1] >> 12) & 0xf
c_hemi = (slot_data[1] >> 16) & 0xf
c_r = (slot_data[1] >> 20) & 0xf
c_g = (slot_data[1] >> 24) & 0xf
c_b = (slot_data[1] >> 28) & 0xf
position_y = (y_base * 0.2 - 200) + (y_height * 0.1)
coord_y.append(position_y)
for i in range(2, 6):
a0 = (slot_data[i] >> 0) & 0xf
a1 = (slot_data[i] >> 4) & 0xf
a2 = (slot_data[i] >> 8) & 0xf
a3 = (slot_data[i] >> 12) & 0xf
return coord_y
def decode_dataset():
Xlist = []
ylist = []
N = SEG_SIDE_LENGTH
BS = 8
PAD = BS - (N * N % BS)
CS = math.ceil(N * N / BS)
print("decode dataset ...")
with bz2.open(DATASET_FILE, "rb") as fp:
while True:
X = fp.read(CS)
y = fp.read(1)
if X == b'' or y == b'':
break
X = [ (ck >> ofs) & 0b1 for ck in X for ofs in range(BS-1, -1, -1) ][:-PAD]
X = np.array(X).reshape(N, N).astype(np.uint8)
y = u(y)
Xlist.append(X)
ylist.append(y)
X = np.array(Xlist)
y = np.array(ylist)
return X, y
def init(self):
geoinputbase.init(self)
#self.log("geoinput")
self.mainCapturer = None #main caputurer
self.switcherCapturer = None
self.switcherFirstClickCapturer = None
self.switcherFirsKeyLastClickAt = 0
self.mod = 0
self.keymap = {
EKey0 : [" ", "0", "\r"],
EKey1 : [".", ",", "?", "!", "@", "'", "-", "1"],
EKey2 : ["ა", "ბ", "ც", "ჩ", "2"],
EKey3 : ["დ", "ე", "ფ", "3"],
EKey4 : ["გ", "ჰ", "ი", "4"],
EKey5 : ["ჯ", "კ", "ლ", "ჟ", "5"],
EKey6 : ["მ", "ნ", "ო", "6"],
EKey7 : ["პ","ქ","რ","ს","ღ","შ","7"],
EKey8 : ["ტ","უ","ვ","თ","8"],
EKey9 : ["წ","ხ","ყ","ზ","ჭ","ძ","9"]
}
for value in self.keymap.values():
for i in range(len(value)):
value[i]=ord(u(value[i]))
def parse_meshes(data):
position = 0
while position < len(data):
mesh_info, position = u('2I', data, position)
mesh_id, mesh_size = mesh_info
parse_mesh(data[position:position + mesh_size], mesh_id)
position += mesh_size
def __init__(self, use_cache=True):
super().__init__()
dataset_file = DATASET_CUSTOM_FILE if os.path.exists(DATASET_CUSTOM_FILE) else DATASET_FILE
print("Use dataset %s" % dataset_file)
with bz2.open(dataset_file, "rb") as fp:
raw = fp.read()
hs = md5(raw)
cache_file = os.path.join(CACHE_DIR, "%s.captcha.gz" % hs)
if os.path.exists(cache_file) and use_cache:
print("Use dataset cache %s" % cache_file)
X, y, labels = joblib.load(cache_file)
self.labels = labels
self.X = X
self.y = y
return
Xlist = []
ylist = []
N = SEG_SIDE_LENGTH
BS = 8
PAD = BS - (N * N % BS)
CS = math.ceil(N * N / BS)
with BytesIO(raw) as fp:
t = tqdm(desc="decode dataset", total=fp.getbuffer().nbytes)
while True:
X = fp.read(CS)
y = fp.read(1)
if X == b'' or y == b'':
break
t.update(CS + 1)
X = np.array([ (ck >> ofs) & 0b1 for ck in X for ofs in range(BS-1, -1, -1) ][:-PAD])
X = 1 - X
y = u(y)
Xlist.append(X)
ylist.append(y)
t.close()
labels = list(sorted(set(ylist)))
assert len(labels) == LABELS_NUM
ixs = np.empty(128, dtype=np.uint8)
for ix, c in enumerate(labels):
ixs[ord(c)] = ix
X = np.array(Xlist, dtype=np.float32).reshape(-1, 1, N, N)
y = np.array([ ixs[ord(c)] for c in ylist ], dtype=np.long)
self.labels = labels
self.X = X
self.y = y
joblib.dump((X, y, labels), cache_file, compress=9)
def parse_main(s):
p = 0
block_id, p = u(I, s, p)
block_size, p = u(I, s, p)
wallmarks_count, p = u(I, s, p)
for i in range(wallmarks_count):
set_count, p = u(I, s, p)
shader, p = utils.parse_string_nul(s, p)
texture, p = utils.parse_string_nul(s, p)
for i in range(set_count):
bounds, p = u(f * 4, s, p)
vertex_count, p = u(I, s, p)
vertices = []
uvs = []
for i in range(vertex_count):
vertex, p = u(f * 3 + I + f * 2, s, p)
vertices.append((vertex[0], vertex[2], vertex[1]))
uvs.append((vertex[4], 1 - vertex[5]))
# generate faces
triangles = []
index = 0
for i in range(vertex_count // 3):
triangles.append((index, index + 2, index + 1))
index += 3
import_utils.crete_mesh(vertices, triangles, None, None, uvs,
texture)
def __init__(self, parent=None, lang='', icon='./images/logo.png'):
super(AboutDlg, self).__init__(parent)
readmefile = utils.do_locale(lang, 'README', '.html')
self.beginDir = parent.beginDir
# En-tête de la fenêtre :
taille = 128
logoLabel = QtGui.QLabel()
logoLabel.setMaximumSize(taille, taille)
logoLabel.setAlignment(QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
logoLabel.setPixmap(QtGui.QPixmap(icon).scaled(taille, taille,
QtCore.Qt.KeepAspectRatio,
QtCore.Qt.SmoothTransformation))
title = utils.u('<h1>{0}</h1><h3>{1}</h3>').format(
QtCore.QCoreApplication.translate(
'AboutDlg', 'About {0}').format(utils.PROGLABEL),
QtCore.QCoreApplication.translate(
'AboutDlg', 'version {0}___{1}').format(utils.PROGVERSION, utils.PROGDATE))
titleLabel = QtGui.QLabel(title)
titleLabel.setAlignment(QtCore.Qt.AlignCenter | QtCore.Qt.AlignVCenter)
titleGroupBox = QtGui.QGroupBox()
titleLayout = QtGui.QHBoxLayout()
titleLayout.addWidget(logoLabel)
titleLayout.addWidget(titleLabel)
titleGroupBox.setLayout(titleLayout)
# Zone d'affichage :
tabWidget = QtGui.QTabWidget()
htmlFile = utils.do_locale(lang, 'translations/README', '.html')
tabWidget.addTab(
FileViewTab(parent=self, htmlFile=htmlFile),
QtCore.QCoreApplication.translate('AboutDlg', 'ReadMe'))
tabWidget.addTab(
FileViewTab(parent=self, txtFile='COPYING'),
QtCore.QCoreApplication.translate('AboutDlg', 'License'))
htmlFile = utils.do_locale(lang, 'translations/CREDITS', '.html')
tabWidget.addTab(
FileViewTab(parent=self, htmlFile=htmlFile),
QtCore.QCoreApplication.translate('AboutDlg', 'Credits'))
# Les boutons :
okButton = QtGui.QPushButton(
QtCore.QCoreApplication.translate('AboutDlg', '&Close'))
okButton.clicked.connect(self.accept)
buttonLayout = QtGui.QHBoxLayout()
buttonLayout.addStretch(1)
buttonLayout.addWidget(okButton)
# Mise en place :
mainLayout = QtGui.QVBoxLayout()
mainLayout.addWidget(titleGroupBox)
mainLayout.addWidget(tabWidget)
mainLayout.addLayout(buttonLayout)
self.setLayout(mainLayout)
self.setWindowTitle(
QtCore.QCoreApplication.translate('AboutDlg', 'About {0}').format(utils.PROGLABEL))
def queue_record(tenant, metric_name, ts, val, props=None):
p = ''
if props:
prop_str = []
for k, v in props.items():
prop_str.append('='.join([k, v]))
p = ','.join(prop_str)
rds.lpush('metrics_queue', ','.join(u([tenant, metric_name, ts, val, p])))
def relative_or_absolute_ts(ts):
from app import u
ts_end = time.time()
if isinstance(ts, str) or isinstance(ts, bytes):
ts_start = u(ts)
if ts_start[0] == '-':
last = ts_start[-1] # -1d, -2h, -1m etc.
r = secs_for[last](int(ts_start[1:-1]))
ts = ts_end - r
return ts
def toggle(self):
simulate_key(8, 8)
if self.currentLang == u'ka':
self.currentLang = u'default'
self.stop()
if self.c('noteSwitch') == 0:
return True
conf(u("default"))
e32.ao_sleep(0.2)
#simulate_key(EKeyHash, EScancodeHash)
else:
self.start()
self.currentLang = u'ka'
if self.c('noteSwitch') == 0:
return True
conf(u('ქართული'))
e32.ao_sleep(0.2)
#simulate_key(EKeyHash, EScancodeHash)
return True
def toggle(self):
simulate_key(8,8)
if self.currentLang == u'ka':
self.currentLang = u'default'
self.stop()
if self.c('noteSwitch') == 0:
return True
conf(u("default"))
e32.ao_sleep(0.2)
#simulate_key(EKeyHash, EScancodeHash)
else:
self.start()
self.currentLang = u'ka'
if self.c('noteSwitch') == 0:
return True
conf(u('ქართული'))
e32.ao_sleep(0.2)
#simulate_key(EKeyHash, EScancodeHash)
return True
def print_dict(_dict,
title=None,
key=None,
value=None,
sep=' ' * 4,
key_sep=' ',
sort=False,
cmp_key=None,
cmp=None,
limit=0):
""" print dict """
if not _dict:
return
_dict = dict((key(k), v) for k, v in _dict.iteritems()) if key else _dict
_dict = dict(
(k, value(v)) for k, v in _dict.iteritems()) if value else _dict
if not sort and not cmp_key and not cmp:
keys, values = _dict.keys(), _dict.values()
else:
_dict = sorted(_dict.iteritems(),
key=cmp_key,
reverse=True if sort < 0 else False,
cmp=cmp)
if limit > 0:
_dict = _dict[:limit]
keys, values = [x for x, _ in _dict], [x for _, x in _dict]
keys = [utils.u(x) if type(x) is str else unicode(x) for x in keys]
key_len = max([utils.chinese_count(x) + len(x) for x in keys])
values = [utils.u(x) if type(x) is str else unicode(x) for x in values]
value_len = max([utils.chinese_count(x) + len(x) for x in values])
tpl = u'{:<%d}' + key_sep + '{:<%d}'
tpl_format = lambda i, x: (tpl % (key_len - utils.chinese_count(x[0][
i]), value_len - utils.chinese_count(x[1][i]))).format(
x[0][i], x[1][i])
step = max(120 / (key_len + value_len + 2 + len(sep)), 1)
lines = [[keys[i:i + step], values[i:i + step]]
for i in range(0, len(keys), step)]
print('=' * 120 + '\n' + title + '\n' + '-' * 120 if title else '=' *
120).encode('utf-8')
print('\n'.join(
sep.join([tpl_format(i, x) for i in range(len(x[0]))])
for x in lines)).encode('utf-8')
def append(self, dest, value):
intvalue = None
if type(value) is unicode:
intvalue = ord(value)
if type(value) is str:
intvalue = ord(u(value))
elif type(value) is int:
intvalue = value
else: return
if intvalue not in dest:
dest.append(intvalue)
def parse_main(dm_data):
position = 0
shader_name, position = utils.parse_string_nul(dm_data, position)
texture_name, position = utils.parse_string_nul(dm_data, position)
dm_info, position = u('IffII', dm_data, position)
flags, min_scale, max_scale, vertex_count, index_count = dm_info
vertices = []
uvs = []
for vertex_id in range(vertex_count):
vertex, position = u('5f', dm_data, position)
loc_x, loc_y, loc_z, uv_x, uv_y = vertex
vertices.append((loc_x, loc_z, loc_y))
uvs.append((uv_x, 1 - uv_y))
triangles = []
for index_id in range(index_count // 3):
face, position = u('3H', dm_data, position)
index_1, index_2, index_3 = face
triangles.append((index_1, index_3, index_2))
return vertices, uvs, triangles, texture_name
def append(self, dest, value):
intvalue = None
if type(value) is unicode:
intvalue = ord(value)
if type(value) is str:
intvalue = ord(u(value))
elif type(value) is int:
intvalue = value
else:
return
if intvalue not in dest:
dest.append(intvalue)
def __repr__(self):
if isinstance(self.data, list):
return self.data.__repr__()
else:
r = []
for d in self.data:
_d = self.data[d]
if isinstance(_d, str) or isinstance(_d, unicode):
r.append(u": ".join([d, u(_d)]))
else:
r.append(u": ".join([d, unicode(_d)]))
return self.__class__.__name__ + u"<" + u", ".join(r) + u">"
def concat_metric_files(metric_name):
chunk_size = 64 * 1024 # 64 kbs
mdir, flist = files_for_metrics(metric_name, 0, 0)
flist = sorted([f for f in u(flist) if '-' not in f])
flist = flist[0:-1] # pop the last file
total = 0
chunk_files = []
for f in u(flist):
if '-' in f:
continue
f = os.path.join(u(mdir), u(f))
fsize = os.path.getsize(os.path.join(f))
total += fsize
if total >= chunk_size:
concat_files(metric_name, mdir, chunk_files)
total = fsize
chunk_files = []
chunk_files.append(f)
def concat_files(metric_name, mdir, files):
dfs = []
for f in u(files):
df = pd.read_csv(f,
header=None,
parse_dates=[0],
index_col=0,
date_parser=date_parser)
print(">>>", f, df.index[0], df.index[-1], len(df))
dfs.append(df)
df = pd.concat(dfs)
df = df.sort_index()
df.index = df.index.astype(np.int64) / 10**9
path = os.path.join(
u(mdir), '{}-{}.csv'.format(int(df.index[0]), int(df.index[-1])))
df.to_csv(path, header=False)
# now we can delete files.
for f in u(files):
os.unlink(f)
return df
def handle(self, index, task):
try:
index, task = task[0], task[1]
url = task['_id']
html = get(u(url))
html = clean_doc(html2doc(html, url=url), return_html=True)
if index == 'cate':
self.domains[task['domain']].parse_cate(task, {'html':html})
elif index == 'album':
pass
elif index == 'page':
self.domains[task['domain']].parse_album(task, {'html':html})
except KeyboardInterrupt:
self.exit()
def handle(self, index, task):
try:
index, task = task[0], task[1]
url = task['_id']
html = get(u(url))
html = clean_doc(html2doc(html, url=url), return_html=True)
if index == 'cate':
self.domains[task['domain']].parse_cate(task, {'html': html})
elif index == 'album':
pass
elif index == 'page':
self.domains[task['domain']].parse_album(task, {'html': html})
except KeyboardInterrupt:
self.exit()
def start_element(name, attrs):
self._path.append(u(name))
self.actObj = MetadataItem(self._path[-1], self._path, "", attrs)
if self.obj == None:
self.obj = self.actObj
else:
found = False
item = getParent(self.obj, self.actObj.path)
if item:
item.addChild(self.actObj)
else:
raise "syntax error"
def start_element(name, attrs):
self._path.append(u(name))
self.actObj = MetadataItem(self._path[-1], self._path, "", attrs)
if self.obj == None:
self.obj = self.actObj
else:
found = False
item = getParent(self.obj, self.actObj.path)
if item:
item.addChild(self.actObj)
else:
raise "syntax error"
def unpack_dataset():
N = SEG_SIDE_LENGTH
BS = 8
PAD = BS - (N * N % BS)
CS = math.ceil(N * N / BS)
rgb_white = np.array([255, 255, 255], dtype=np.uint8)
rgb_black = np.array([0, 0, 0], dtype=np.uint8)
cnts = np.zeros(128, dtype=np.int)
ignored = get_trash_segs_md5()
with bz2.open(DATASET_FILE, "rb") as fp:
raw = fp.read()
with BytesIO(raw) as fp:
t = tqdm(desc="unpack dataset", total=fp.getbuffer().nbytes)
while True:
X = fp.read(CS)
y = fp.read(1)
if X == b'' or y == b'':
break
t.update(CS + 1)
X = np.array([(ck >> ofs) & 0b1 for ck in X
for ofs in range(BS - 1, -1, -1)][:-PAD])
im = np.array([rgb_white if b else rgb_black for b in X],
dtype=np.uint8).reshape(N, N, 3)
y = u(y)
k = ord(y)
cnts[k] += 1
hs = md5(im.tobytes())
if hs in ignored:
continue
y = y + "_" if y.islower() else y
folder = os.path.join(DECODED_DATASET_DIR, y)
filename = "%s_%06d.png" % (y, cnts[k])
path = os.path.join(folder, filename)
mkdir(folder)
cv2.imwrite(path, im)
t.close()
def extend(self, key_p, value):
if type(key_p) is str:
key = ord(u(key_p))
elif type(key_p) is int:
key = key_p
else:
return
if key not in self.keymapkbd:
self.keymapkbd[key] = tuple([])
tmp = list(self.keymapkbd[key])
if type(value) is list:
for v in value:
self.append(tmp, v)
else:
self.append(tmp, value)
self.keymapkbd[key] = tuple(tmp)
def extend(self, key_p, value):
if type(key_p) is str:
key = ord(u(key_p))
elif type(key_p) is int:
key = key_p
else:
return
if key not in self.keymapkbd:
self.keymapkbd[key] = tuple([])
tmp = list(self.keymapkbd[key])
if type(value) is list:
for v in value:
self.append(tmp, v)
else:
self.append(tmp, value)
self.keymapkbd[key] = tuple(tmp)
def parse_header(data, coord_y):
position = 0
header_data, position = u('IIiiII', data, position)
version, object_count, offset_x, offset_z, size_x, size_z = header_data
slots_coords = []
slots_count = size_x * size_z
column = 0 - offset_x + size_x
row = 1 - offset_z
for slot in range(slots_count):
if slot % size_x != 1:
column += 1
slots_coords.append((2 * (column), 2 * (row), coord_y[slot]))
else:
column += 1 - size_x
row += 1
slots_coords.append((2 * (column), 2 * (row), coord_y[slot]))
return slots_coords
def get(self):
if "query" not in request.args:
abort(404, message="parameter `query` doestn't exist")
query = u(request.args["query"])
# logging.debug("new query '%s'" % (query))
liststr = ClearAndSegment(query)
# print liststr
liststr = [word.encode("UTF-8") for word in liststr if word not in stops]
score = f.singlejudge(liststr)
# print score
body = {}
if score > threshold:
body = {"spam": "True"}
else:
body = {"spam": "False"}
resp = make_response(jsonify(body))
return resp
def query_range(tenant):
metric_names, start, end, step = parse_query_args(tenant)
result = []
for metric_name in metric_names:
vals = load_metrics(metric_name, start, end, step)
result.append({
"metric": {
"__name__": u(metric_name),
},
"values": vals
})
return jsonify({
"status" : "success",
"data" : {
"resultType" : "matrix",
"result" : result
}
})
def process_line(valln):
val = u(valln)
values = val.split(',')
tenant, metric_name, ts, val = values[0:4]
if len(values) == 4:
metric_props = {}
else:
mp = values[4:]
metric_props = {}
for m in mp:
if m:
k, v = m.split('=')
metric_props[k] = v
ts = str_int_or_float_value(ts)
val = str_int_or_float_value(val)
save_record(tenant, metric_name, ts, val, metric_props)
def get(self):
if 'query' not in request.args:
abort(404, message="parameter `query` doestn't exist")
query = u(request.args['query'])
#logging.debug("new query '%s'" % (query))
liststr = ClearAndSegment(query)
#print liststr
liststr = [
word.encode('UTF-8') for word in liststr if word not in stops
]
score = f.singlejudge(liststr)
#print score
body = {}
if score > threshold:
body = {'spam': 'True'}
else:
body = {'spam': 'False'}
resp = make_response(jsonify(body))
return resp
def getXMLMetaString(self, obj=None, indent=0, ret=""):
if not obj:
obj = self.obj
ret += '\r\n' + (' ' * indent) + '<' + u(obj.path[-1])
if len(obj.attr) > 0:
for item in obj.attr:
ret += ' ' + u(item) + '="' + u(obj.attr[item]) + '"'
ret += '>'
if len(obj.children) > 0:
for item in obj.children:
ret += self.getXMLMetaString(item, indent + 1)
ret += '\r\n' + (' ' * indent) + '</' + u(obj.path[-1]) + '>'
else:
ret += u(obj.value)
ret += '</' + u(obj.path[-1]) + '>'
return ret
def getXMLMetaString(self, obj=None, indent=0, ret=""):
if not obj:
obj = self.obj
ret += '\r\n' + (' ' * indent) + '<' + u(obj.path[-1])
if len(obj.attr) > 0:
for item in obj.attr:
ret += ' ' + u(item) + '="' + u(obj.attr[item]) + '"'
ret += '>'
if len(obj.children) > 0:
for item in obj.children:
ret += self.getXMLMetaString(item, indent + 1)
ret += '\r\n' + (' ' * indent) + '</' + u(obj.path[-1]) + '>'
else:
ret += u(obj.value)
ret += '</' + u(obj.path[-1]) + '>'
return ret
def chinese_count(text): """ count chinese char """ return len(RE_CHINESE_CHAR.findall(utils.u(text)))
def english_word_count(text): """ count english word """ return len(RE_ENGLISH_WORD.findall(utils.u(text)))
def word_count(text): """ count word """ return len(RE_WORD.findall(utils.u(text)))
def get_chinese(text): return RE_CHINESE_CHAR.findall(utils.u(text))
##############################
f.test(threshold)
print "tar = %f" % (f.tar)
print "trr = %f" % (f.trr)
print "accuracy = %f" % (f.accuracy)
##############################
# 3.single judge
##############################
stopwords_file = configs['stopwords_file']
fstop = open(stopwords_file)
totalStop = fstop.readlines()
fstop.close()
stops = []
for s in totalStop:
s = s.strip()
stops.append(s)
query = '赚钱test宝妈tes日赚学生兼职*.@打字员'
query = u(query)
print query
listquery = ClearAndSegment(query)
#print listquery
listquery = [word.encode('UTF-8') for word in listquery if word not in stops]
if f.singlejudge(listquery) > threshold:
print 'WARNING: this is a spam message'
else:
print 'this message is harmless'
def extract_token(token, value):
#~ print "extrayendo %s - %s"%(token, value)
try:
xp = xpath.get_xpath(u(token))
except UnicodeDecodeError:
#~ print "unicode"
return False
if xp is None:
#~ print "xp"
return False
#~ print "XP", xp
extract = xpath.extract(xp)
if not extract:
#~ print "no extract", token, xp
return False
#~ print ".."
#~ print element_2_str(extract)
#~ print "TOKEN-EXTRACT", token.strip(), extract.strip()
if len(extract) > 0 or not token.strip() == extract.strip():
#~ print "[%s][%s]"%(token, extract[0])
ok = True
if xpath.last_expansive:
ok = False
try:
if token.strip() in element_2_str(extract):
#es correcto, probablemente la descripcion
ok = True
else:
#~ self.logger.warning("else last_expansive")
return False
except:
return False
if not ok:
if not token.strip() in extract.strip():
print("No coincide %s[%s] para %s"%(xp, extract, token))
self.logger.error("No coincide %s[%s] para %s"%(xp, extract, token))
raise Exception("Incoherencia xpath")
#~ self.logger.warning("No se puede extrar el xpath de %s en %s"%(token, url) )
return False
id_m = m
if m == "tags":
tg = is_tag(value)
if tg:
id_m = tg.split("_")[0]
if id_m in _metadata and _metadata[id_m]['xpath'] != xp and id_m != "category":
duplicated.append(id_m)
#Para el metadata "language" no se guardan xpath de enlaces ya que suelen ser selectores de idioma de la página
if id_m == "language" and "/a/" in xp:
return False
#h1 solo para el title
if id_m != "title" and "/h1" in xp:
return False
if "'tab-main'" in xp and id_m == "subcategory":
print
print
print
print url
print id_m
print _metadata
print value
print xp
exit()
#No se guarda nada que cuelgue de comentarios, script o style
invalid = ["comment", "script", "style", "select"]
if not any([w in xp for w in invalid]) :
_metadata[id_m] = {"value" : value, "xpath" : xp}
def __init__(self, parent=None):
super(OtherPage, self).__init__(parent)
self.main = parent.main
# configuration de la variable screenShotDelay :
groupBoxScreenShotDelay = QtGui.QGroupBox('')
title = QtGui.QApplication.translate('main', 'ScreenShotDelay')
title2 = QtGui.QApplication.translate('main', 'in milliseconds')
titleLabel = QtGui.QLabel(utils.u('<p align="center"><b>{0}</b> ({1})</p>').format(title, title2))
integerLabel = QtGui.QLabel(QtGui.QApplication.translate('main',
'Enter a value between {0} and {1}:').format(10, 10000))
self.integerSpinBoxScreenShotDelay = QtGui.QSpinBox()
self.integerSpinBoxScreenShotDelay.setRange(10, 10000)
self.integerSpinBoxScreenShotDelay.setSingleStep(100)
self.integerSpinBoxScreenShotDelay.setValue(self.main.screenShotDelay)
hLayout = QtGui.QHBoxLayout()
hLayout.addWidget(integerLabel)
hLayout.addWidget(self.integerSpinBoxScreenShotDelay)
vLayout = QtGui.QVBoxLayout()
vLayout.addWidget(titleLabel)
vLayout.addLayout(hLayout)
vLayout.addStretch()
groupBoxScreenShotDelay.setLayout(vLayout)
# configuration de la variable attachDistance :
groupBoxAttachDistance = QtGui.QGroupBox('')
title = QtGui.QApplication.translate('main', 'AttachDistance')
title2 = QtGui.QApplication.translate('main', 'between lines or points and ruler or square')
titleLabel = QtGui.QLabel(utils.u('<p align="center"><b>{0}</b> ({1})</p>').format(title, title2))
integerLabel = QtGui.QLabel(QtGui.QApplication.translate('main',
'Enter a value between {0} and {1}:').format(0, 100))
self.integerSpinBoxAttachDistance = QtGui.QSpinBox()
self.integerSpinBoxAttachDistance.setRange(0, 100)
self.integerSpinBoxAttachDistance.setSingleStep(1)
self.integerSpinBoxAttachDistance.setValue(self.main.attachDistance)
hLayout = QtGui.QHBoxLayout()
hLayout.addWidget(integerLabel)
hLayout.addWidget(self.integerSpinBoxAttachDistance)
vLayout = QtGui.QVBoxLayout()
vLayout.addWidget(titleLabel)
vLayout.addLayout(hLayout)
vLayout.addStretch()
groupBoxAttachDistance.setLayout(vLayout)
# configuration de l'impression :
groupBoxPrintConfig = QtGui.QGroupBox('')
title = QtGui.QApplication.translate('main', 'PrintConfig')
titleLabel = QtGui.QLabel(utils.u('<p align="center"><b>{0}</b></p>').format(title))
# le mode d'impression (FullPage ou TrueSize) :
title = QtGui.QApplication.translate('main', 'PrintMode')
groupBoxPrintMode = QtGui.QGroupBox(title)
self.radioPrintFullPage = QtGui.QRadioButton(QtGui.QApplication.translate('main', 'FullPage'))
self.radioPrintTrueSize = QtGui.QRadioButton(QtGui.QApplication.translate('main', 'TrueSize'))
if self.main.printMode == 'FullPage':
self.radioPrintFullPage.setChecked(True)
else:
self.radioPrintTrueSize.setChecked(True)
vLayout = QtGui.QVBoxLayout()
vLayout.addWidget(self.radioPrintFullPage)
vLayout.addWidget(self.radioPrintTrueSize)
groupBoxPrintMode.setLayout(vLayout)
# orientation (Portrait ou Landscape) :
title = QtGui.QApplication.translate('main', 'Orientation')
groupBoxPrintOrientation = QtGui.QGroupBox(title)
self.radioPrintPortrait = QtGui.QRadioButton(QtGui.QApplication.translate('main', 'Portrait'))
self.radioPrintLandscape = QtGui.QRadioButton(QtGui.QApplication.translate('main', 'Landscape'))
if self.main.printOrientation == 'Portrait':
self.radioPrintPortrait.setChecked(True)
else:
self.radioPrintLandscape.setChecked(True)
vLayout = QtGui.QVBoxLayout()
vLayout.addWidget(self.radioPrintPortrait)
vLayout.addWidget(self.radioPrintLandscape)
groupBoxPrintOrientation.setLayout(vLayout)
# on agence tout ça :
hLayout = QtGui.QHBoxLayout()
hLayout.addWidget(groupBoxPrintMode)
hLayout.addWidget(groupBoxPrintOrientation)
vLayout = QtGui.QVBoxLayout()
vLayout.addWidget(titleLabel)
vLayout.addLayout(hLayout)
# les explications :
bigEditorPrintMode = QtGui.QTextEdit()
bigEditorPrintMode.setReadOnly(True)
bigEditorPrintMode.setText(
QtGui.QApplication.translate('main', ''
'<p align=left><b>FullPage: </b>'
'printing will be adapted to the dimensions of the page.</p>'
'<p></p>'
'<p align=left><b>TrueSize: </b>'
'the printed document comply with the dimensions of your figures.</p>'
'<p></p>'))
vLayout.addWidget(bigEditorPrintMode)
vLayout.addStretch(1)
groupBoxPrintConfig.setLayout(vLayout)
# mise en place :
layout = QtGui.QVBoxLayout()
layout.addWidget(groupBoxScreenShotDelay)
layout.addWidget(groupBoxAttachDistance)
layout.addWidget(groupBoxPrintConfig)
self.setLayout(layout)
def createToolBars(self):
# la barre de base (quitter, aide, ...)
self.toolBarBase = QtGui.QToolBar(QtGui.QApplication.translate("main", "Base Bar"))
self.toolBarBase.setObjectName("toolBarBase")
self.addToolBar(QtCore.Qt.TopToolBarArea, self.toolBarBase)
# la barre des screenshots et autres fonds
self.toolBarBackground = QtGui.QToolBar(QtGui.QApplication.translate("main", "Background Bar"))
self.toolBarBackground.setObjectName("toolBarBackground")
self.addToolBar(QtCore.Qt.TopToolBarArea, self.toolBarBackground)
# la barre des instruments
self.toolBarInstruments = QtGui.QToolBar(QtGui.QApplication.translate("main", "Tools Bar"))
self.toolBarInstruments.setObjectName("toolBarInstruments")
self.addToolBar(QtCore.Qt.TopToolBarArea, self.toolBarInstruments)
# la barre des outils de dessin (et son comboboxPens)
self.toolBarDraw = QtGui.QToolBar(QtGui.QApplication.translate("main", "Draw Bar"))
self.toolBarDraw.setObjectName("toolBarDraw")
self.addToolBar(QtCore.Qt.TopToolBarArea, self.toolBarDraw)
# la barre de réglage des dessins
self.toolBarDrawConfig = QtGui.QToolBar(QtGui.QApplication.translate("main", "Draw Config Bar"))
self.toolBarDrawConfig.setObjectName("toolBarDrawConfig")
self.addToolBar(QtCore.Qt.TopToolBarArea, self.toolBarDrawConfig)
# la barre des couleurs et tailles persos
self.toolBarColorsSizes = QtGui.QToolBar(QtGui.QApplication.translate("main", "Custom colors and sizes Bar"))
self.toolBarColorsSizes.setObjectName("toolBarColorsSizes")
self.addToolBar(QtCore.Qt.TopToolBarArea, self.toolBarColorsSizes)
self.toolBarsList = (
self.toolBarBase,
self.toolBarBackground,
self.toolBarInstruments,
self.toolBarDraw,
self.toolBarDrawConfig,
self.toolBarColorsSizes,
)
self.toolBars = {
# la barre de base (quitter, aide, ...)
self.toolBarBase: (
self.actionQuit,
"SEPARATOR",
self.actionFileOpen,
self.actionFileReload,
self.actionFileGoPrevious,
self.actionFileGoNext,
"SEPARATOR",
self.actionFileSave,
self.actionFileSaveAs,
self.actionPrint,
self.actionPrintDirect,
self.actionExportPdf,
self.actionExportSvg,
"SEPARATOR",
self.actionConfigure,
self.actionMinimize,
self.actionCreateDesktopFileLinux,
"SEPARATOR",
self.actionHelp,
self.actionAbout,
),
# la barre des screenshots et autres fonds
self.toolBarBackground: (
self.actionNewScreenshot,
self.actionWhitePage,
self.actionPointsPage,
self.actionGridPage,
self.actionBackGround,
),
# la barre des instruments
self.toolBarInstruments: (
self.actionShowRuler,
self.actionShowSquare,
self.actionShowSquareNotGraduated,
self.actionShowProtractor,
self.actionShowCompass,
"SEPARATOR",
self.actionUnitsLock,
self.actionUnitsSave,
self.actionUnitsRestore,
self.actionUnitsInit,
self.actionLockInstruments,
self.actionShowFalseCursor,
),
#
self.toolBarDraw: (
self.actionSelect,
"SEPARATOR",
self.actionPens,
self.actionDrawLine,
self.actionDrawCurve,
self.actionYellowHighlighterPen,
self.actionGreenHighlighterPen,
self.actionPinkHighlighterPen,
self.actionBlueHighlighterPen,
"SEPARATOR",
self.actionAddText,
self.actionAddPoint,
self.actionAddPixmap,
self.actionPaste,
),
# la barre de réglage des dessins
self.toolBarDrawConfig: (
self.actionPenColor,
self.actionPenWidth,
self.actionPenStyle,
self.actionChooseFont,
"SEPARATOR",
self.actionEditText,
"SEPARATOR",
self.actionUndo,
self.actionRedo,
"SEPARATOR",
self.actionRemoveSelected,
self.actionRemoveLast,
self.actionEraseAll,
),
# la barre des couleurs et tailles persos
self.toolBarColorsSizes: (
self.actionEditCustomColors,
self.actionCustomColor1,
self.actionCustomColor2,
self.actionCustomColor3,
self.actionCustomColor4,
self.actionCustomColor5,
self.actionCustomColor6,
self.actionCustomColor7,
self.actionCustomColor8,
self.actionCustomColor9,
"SEPARATOR",
self.actionEditCustomSizes,
self.actionCustomSize1,
self.actionCustomSize2,
self.actionCustomSize3,
self.actionCustomSize4,
self.actionCustomSize5,
),
}
self.allwaysActions = (self.actionQuit, self.actionConfigure, self.actionHelp, self.actionAbout)
settings = QtCore.QSettings(utils.PROGNAME, "config")
defaultSettings = QtCore.QSettings("libs/config.conf", QtCore.QSettings.IniFormat)
for toolBar in self.toolBarsList:
for action in self.toolBars[toolBar]:
if isinstance(action, QtGui.QWidgetAction):
actionName = action.defaultWidget().toolTip()
actionText = action.defaultWidget().toolTip()
elif isinstance(action, QtGui.QAction):
actionName = action.objectName()
actionText = action.text()
else:
actionName = ""
if action in self.actionCustomColors:
actionText = utils.u("{0} {1}").format(
QtGui.QApplication.translate("main", "CustomColor"), self.actionCustomColors.index(action) + 1
)
if action in self.actionCustomSizes:
actionText = utils.u("{0} {1}").format(
QtGui.QApplication.translate("main", "CustomSize"), self.actionCustomSizes.index(action) + 1
)
if (action not in self.allwaysActions) and (actionName != ""):
visible = settings.value(utils.u("{0}/{1}").format(toolBar.objectName(), actionName), 1)
if int(visible) == 0:
action.setVisible(False)
kidVisible = settings.value(
utils.u("toolBarKid/{0}").format(actionName),
defaultSettings.value(utils.u("toolBarKid/{0}").format(actionName)),
)
if kidVisible == None:
kidVisible = 0
if int(kidVisible) == 0:
kidVisible = False
else:
kidVisible = True
self.actionsState[action] = [actionText, action.isVisible(), kidVisible]
self.reloadToolBars()
def createTrainAndTestData(strpos, strneg, trainRate, stopWords):
fpos = open(strpos)
totalPos = fpos.readlines()
fneg = open(strneg)
totalNeg = fneg.readlines()
#trainRate = 0.9999
trainPosNum = int(trainRate * len(totalPos))
trainNegNum = int(trainRate * len(totalNeg))
trainPosIndex = random.sample(range(len(totalPos)),trainPosNum)
trainNegIndex = random.sample(range(len(totalNeg)),trainNegNum)
trainPosArray = []
testPosArray = []
trainNegArray = []
testNegArray = []
fstop = open(stopWords)
totalStop = fstop.readlines()
stops = []
for s in totalStop:
s = s.strip()
s = u(s)
stops.append(s)
index = 0
for i in range(len(totalPos)):
str = totalPos[i]
str = str.strip()
if str == '':
continue
#print str
str = u(str)
str = ClearAndSegment(str)
str = [word for word in str if word not in stops]
str = '/'.join(str)
str = str + '\n'
if not str.strip():
continue
index +=1
if index%1000 == 0:
print 'Adding positive data: %d' % (index)
if i in trainPosIndex:
trainPosArray.append(str)
else:
testPosArray.append(str)
print '\n'
index = 0
for i in range(len(totalNeg)):
str = totalNeg[i]
str = str.strip()
if str == '':
continue
str = u(str)
str = ClearAndSegment(str)
str = [word for word in str if word not in stops]
str = '/'.join(str)
str = str + '\n'
if not str.strip():
continue
index +=1
if index%1000 == 0:
print 'Adding negative data: %d' % (index)
if i in trainNegIndex:
trainNegArray.append(str)
else:
testNegArray.append(str)
print '\n'
print "len of trainPos is %d\n" % (len(trainPosArray))
print "len of trainNeg is %d\n" % (len(trainNegArray))
print "len of testPos is %d\n" % (len(testPosArray))
print "len of testNeg is %d\n" % (len(testNegArray))
ftrainPos = open(trainPos, "w")
ftrainNeg = open(trainNeg, "w")
ftestPos = open(testPos, "w")
ftestNeg = open(testNeg, "w")
#for ele in trainPosArray:
# ftrainPos.write(ele)
ftrainPos.writelines(trainPosArray)
ftrainNeg.writelines(trainNegArray)
ftestPos.writelines(testPosArray)
ftestNeg.writelines(testNegArray)
fpos.close()
fneg.close()
ftrainPos.close()
ftestPos.close()
ftrainNeg.close()
ftestNeg.close()
fstop.close()
def english_count(text): """ count english char """ return len(RE_ENGLISH_CHAR.findall(utils.u(text)))