def child_by_text(self, txt, **kwargs):
if "allow_scroll_search" in kwargs:
allow_scroll_search = kwargs.pop("allow_scroll_search")
name = self.jsonrpc.childByText(self.selector, Selector(**kwargs),
txt, allow_scroll_search)
else:
name = self.jsonrpc.childByText(self.selector, Selector(**kwargs),
txt)
return AutomatorDeviceNamedUiObject(self.device, name)
async def test_select_both_push(self):
"""Push and popping should perform the first available op"""
sel1, sel2 = Selector(), Selector()
async with Channel(1) as chan1, Channel(1) as chan2:
# c1 will be empty (cannot pop)
sel1.pop(chan1)
sel1.push(chan2, 10)
val = await sel1.gather()
self.assertEqual(val, (1, True))
# c2 will be empty (cannot pop)
self.assertEqual(await chan2.pop(), 10)
sel2.push(chan1, 20)
sel2.pop(chan2)
val = await sel2.gather()
self.assertEqual(val, (0, True))
async def test_select_pop(self):
"""Popping on two channels should return the one with data"""
sel1, sel2 = Selector(), Selector()
async with Channel(1) as chan1, Channel(1) as chan2:
# Pop from C2 first
sel1.pop(chan1)
sel1.pop(chan2)
await chan2.push(10)
val = await sel1.gather()
self.assertEqual(val, (1, 10))
# Now pop from C1
sel2.pop(chan1)
sel2.pop(chan2)
await chan1.push(20)
val = await sel2.gather()
self.assertEqual(val, (0, 20))
def parse( self, url, html ):
# Create XPath
sel = Selector( html )
# Return State, New Link
return True, []
def __init__(self, arg):
# setting
self.name_map = arg.name_map
self.opt = Setting()
# pygame
pygame.init()
self.screen = pygame.display.set_mode(self.opt.WINDOW_SIZE)
pygame.display.set_caption(self.opt.TITLE % self.name_map)
self.clock = pygame.time.Clock()
self.set_grill_surface()
# data
self.path = os.path.dirname(__file__)
self.path_img = os.path.join(self.path, "..", "src", "img")
file = self.name_map + self.opt.LUA_FORMAT
self.path_maps = os.path.join(self.path, "..", "src", "maps", file)
self.create = not os.path.exists(self.path_maps)
# objects
self.builder = Builder(self)
self.converter = Converter(self)
self.images = Images(self)
self.map = Map(self)
self.selector = Selector(self)
self.toolbar = Toolbar(self)
# loop
self.saved = True
self.loop()
pygame.quit()
def _setSelectorText(self, selectorText):
"""
:param selectorText:
comma-separated list of selectors or a tuple of
(selectorText, dict-of-namespaces)
:exceptions:
- :exc:`~xml.dom.NamespaceErr`:
Raised if the specified selector uses an unknown namespace
prefix.
- :exc:`~xml.dom.SyntaxErr`:
Raised if the specified CSS string value has a syntax error
and is unparsable.
- :exc:`~xml.dom.NoModificationAllowedErr`:
Raised if this rule is readonly.
"""
self._checkReadonly()
# might be (selectorText, namespaces)
selectorText, namespaces = self._splitNamespacesOff(selectorText)
try:
# use parent's only if available
namespaces = self.parentRule.parentStyleSheet.namespaces
except AttributeError:
pass
wellformed = True
tokenizer = self._tokenize2(selectorText)
newseq = []
expected = True
while True:
# find all upto and including next ",", EOF or nothing
selectortokens = self._tokensupto2(tokenizer, listseponly=True)
if selectortokens:
if self._tokenvalue(selectortokens[-1]) == ',':
expected = selectortokens.pop()
else:
expected = None
selector = Selector((selectortokens, namespaces), parent=self)
if selector.wellformed:
newseq.append(selector)
else:
wellformed = False
self._log.error(u'SelectorList: Invalid Selector: %s' %
self._valuestr(selectortokens))
else:
break
# post condition
if u',' == expected:
wellformed = False
self._log.error(u'SelectorList: Cannot end with ",": %r' %
self._valuestr(selectorText))
elif expected:
wellformed = False
self._log.error(u'SelectorList: Unknown Syntax: %r' %
self._valuestr(selectorText))
if wellformed:
self.seq = newseq
def get_selector():
# selector columns
dataset = get_data(selector=False)
train_data = dataset[dataset['rate'] > 0.0]
test_data = dataset[dataset['rate'] < 0.0]
train_data.reset_index(inplace=True, drop=True)
test_data.reset_index(inplace=True, drop=True)
columns = train_data.columns
remove_columns = ['sample_id', 'rate']
features_columns = [
column for column in columns if column not in remove_columns
]
x_train = train_data[features_columns]
y_train = train_data['rate']
selector = Selector()
select_columns = selector.get_select_features(x_train, y_train)
select_columns.extend(remove_columns)
remove_columns = [
column for column in train_data.columns if column not in select_columns
]
train_data = train_data.drop(columns=remove_columns)
test_data = test_data.drop(columns=remove_columns)
dataset = pd.concat([train_data, test_data])
selector_name = os.path.join(ETLDATA_PATH, 'selector_features.csv')
dataset.to_csv(selector_name, index=False)
def _parse_properties(self, htmltree):
# Populate properties from selectors
_properties = {}
config_props = self.config['properties']
# Search for each property option
for property in self._selector_properties:
if property not in config_props:
logging.info("No config found for {}".format(property))
continue
logging.info('Working on {}'.format(property))
args = {
'selector': config_props[property],
}
if property in self._dates:
args['is_date'] = True
selector = Selector(**args)
logging.info("Using selector \"{}\" with selector type {}".format(
selector.selector,
selector.SelectorType.__class__.__name__,
))
result_text = selector.result(htmltree)
logging.info('Found prop value {}'.format(result_text))
_properties[property] = result_text
return _properties
def test_pause(self):
"""
Test Selector with stop_condition=gt10 and pause_condition=is_even
"""
sel = Selector(gt10, is_even, self.gens)
expected = [5, 1, 7, 3, 9, 5, 7, 9]
self.assertEqual(expected, list(sel))
def test_no_pause(self):
"""
Test Selector with stop_condition=gt10 and no pause condition
"""
sel = Selector(gt10, None, self.gens)
expected = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 5, 6, 7, 8, 9, 10]
self.assertEqual(expected, list(sel))
def test_no_stop(self):
"""
Test Selector without stop or pause conditions
"""
sel = Selector(Selector.false, None, [gen1(), gen2()])
expected = list(gen1()) + list(gen2())
self.assertEqual(expected, list(sel))
def xpath(self, xpath_expr: str):
"""
xpath语法
:return:
"""
selector = Selector(self.content, xpath_expr=xpath_expr)
return selector
def apply(self, query):
"""
引擎构建好后，即可解析query
"""
matched_items, special_post = self.searcher.search_match(query)
sel = Selector(matched_items, special_post, self.rule_info)
return sel.apply(query)
def get_processing(self):
# 最终处理数据生成特征的函数
dataset = self._get_data_plusaugmentation()
boolean_columns = self._get_boolean_columns(dataset)
remove_columns = ['id', 'score']
num_columns = list() # 摘取其余的特征
for column in dataset.columns:
if column in remove_columns:
continue
if column in boolean_columns:
continue
num_columns.append(column)
# 异常字段处理：手动分箱
abnormal_columns = [
'online_shopping_num', 'logistics_num', 'financing_num',
'video_num', 'airplant_num', 'train_num', 'travel_num'
]
abnormal_encoder_columns = list()
for column in abnormal_columns:
encoder_column = f'{column}_encoder'
dataset[encoder_column] = dataset[column].apply(
self._get_abnormal_label)
abnormal_encoder_columns.append(encoder_column)
dataset = pd.get_dummies(dataset, columns=abnormal_encoder_columns)
#_get_abnormal_label函数只是把数据变成1234 但是要通过get_dummies变成哑变量0001
# 缺失值单独抽离特征：无效 这怎么可能有效啊！无语子
# 那既然这样无效的话 用什么办法处理缺失值呢？
# 是认为 除了boolean特征和刚刚处理过的app使用次数特征之外 其余的0值就是missing data啦
# for column in num_columns:
# # abnormal已处理过，continue
# if column in abnormal_columns:
# continue
# column_name = f'{column}_missing'
# dataset[column_name] = dataset[column].apply(self._get_missing_value)
# 将bool类型重新组合 也就是交互特征啦
# 生成的特征有点过多了 可以考虑去掉
# dataset = self._recombine_boolean_columns(dataset, boolean_columns)
# ??看看是啥特征 没看懂是在干嘛
# dataset = self._data_encoder(dataset, ['surfing_time', 'age'])
# 业务逻辑特征
dataset = self._get_operation_features(dataset)
if self.selector:
train_data = dataset[dataset['score'] > 0]
y_data = train_data['score']
x_data = train_data.drop(columns=['id', 'score'])
# 选特征
select_features = Selector(
ascending=self.ascending).get_select_features(x_data, y_data)
select_features.extend(['id', 'score'])
dataset = dataset[select_features]
return dataset
async def test_select_closed(self):
"""Select on a closed channel should return None"""
sel = Selector()
chan = Channel(2)
await chan.close()
sel.pop(chan)
val = await sel.gather()
self.assertEqual(val, (0, None))
def to(obj, *args, **kwargs):
if len(args) >= 2 or "x" in kwargs or "y" in kwargs:
drag_to = lambda x, y, steps=100: self.jsonrpc.dragTo(
self.selector, x, y, steps)
else:
drag_to = lambda steps=100, **kwargs: self.jsonrpc.dragTo(
self.selector, Selector(**kwargs), steps)
return drag_to(*args, **kwargs)
def __view_beside(self, onsideof, **kwargs):
bounds = self.info["bounds"]
min_dist, found = -1, None
for ui in AutomatorDeviceObject(self.device, Selector(**kwargs)):
dist = onsideof(bounds, ui.info["bounds"])
if dist >= 0 and (min_dist < 0 or dist < min_dist):
min_dist, found = dist, ui
return found
def run(self):
print 'hi, thread running'
selector_obj = Selector()
file_handler = FileHandler()
#this won't work unless you make a separate thread.
new_file = file_handler.monitor()
self.results = selector_obj.validate(new_file)
return
def main():
user_data = input().split()
user = User(user_data[0], user_data[1], int(user_data[2]))
book_data = Receiver(user).collect_books()
chosen_book = Selector(book_data).select_book()
print(chosen_book)
def test_stop(self):
"""
Test .stop()
"""
sel = Selector(gt10, is_even, self.gens)
sel.stop()
self.assertEqual([], sel.gens)
self.assertIsNone(sel.started)
self.assertIsNone(sel.curr)
def __prepareset(self, newSelector):
# used by appendSelector and __setitem__
self._checkReadonly()
if not isinstance(newSelector, Selector):
newSelector = Selector(newSelector)
if newSelector.wellformed:
return newSelector
def __init__(self, word_vec, weight=None):
super(LatentRE, self).__init__()
''' load encoder '''
if Config.encoder == "bert":
self.encoder = Bert()
else:
self.encoder = TextRepre(word_vec)
self.selector = Selector()
self.loss = Loss(weight)
def test_select_on(self):
"""
Test .select_on decorator
"""
sel = Selector(gt10)
sel.select_on(gen1)
self.assertEqual(1, len(sel.gens))
sel_gen = sel.gens[0]
#self.assertEqual(gen1.__doc__, sel_gen.__doc__)
self.assertEqual(gen1.__name__, sel_gen.__name__)
def __prepareset(self, newSelector, namespaces=None):
"Used by appendSelector and __setitem__"
if not namespaces:
namespaces = {}
self._checkReadonly()
if not isinstance(newSelector, Selector):
newSelector = Selector((newSelector, namespaces), parent=self)
if newSelector.wellformed:
newSelector._parent = self # maybe set twice but must be!
return newSelector
def __init__(self, ip, port, ReqeustHandle, **kwargs):
self.loop = Selector()
self.ip = ip
self.port = port
self.RequestHandle = ReqeustHandle
self.kwargs = kwargs
self.server = socket.socket()
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
def _parse_entries(self, htmltree):
_entries = []
logging.info("Working on entries")
config_entries = self.config['entries']
entries_selector = Selector(config_entries, multiple=True)
logging.info("Using selector \"{}\" with selector type {}".format(
entries_selector.selector,
entries_selector.SelectorType.__class__.__name__,
))
result_entries = entries_selector.result(htmltree)
logging.info('Got {} entries'.format(len(result_entries)))
for entry in result_entries:
logging.info('Working on entry {}'.format(entry))
tmp_entry = {}
for property in self._entry_selector_properties:
if property not in config_entries:
logging.info("No config found for {}".format(property))
continue
logging.info('Working on prop {}'.format(property))
args = {
'selector': config_entries[property],
}
if property in self._dates:
args['is_date'] = True
selector = Selector(**args)
logging.info(
"Using selector \"{}\" with selector type {}".format(
selector.selector,
selector.SelectorType.__class__.__name__,
))
tmp_entry[property] = selector.result(entry)
_entries.append(tmp_entry)
return _entries
def Selecionar():
print('Informe o tipo de Consulta: C - COMPLETA | A - ATUALIZAÇÃO: ',
end='')
tipo = str(input())
print('Informe a Data INICIAL para a Consulta [YYYY-MM-DD]: ', end='')
data_inicio = str(input())
print('Informe a Data FINAL para a Consulta [YYYY-MM-DD]: ', end='')
data_final = str(input())
selecao = Selector(
) #Faz a seleção do conteudo dos tweets e insere em Banco de dados.
selecao.select(tipo, data_inicio, data_final)
def get_app(conf=fuse.config):
# Create the mapper object: the core decision-maker
mapper = Selector()
# Middleware called in this order, before the mapper
application = muddleware.ExceptionHandler(mapper)
# Set up the various components
fuse.api.APIWrapper(conf, fuse.db.get_database(conf), mapper)
return application
def __init__(self, conf):
self.conf = conf
self.log = conf.get('log')
self.site_statistic = SiteStatistics(conf['site_task_collect_db'])
self.seed_statistic = SeedStatistics(conf['seed_task_collect_db'])
self.scheduler = SiteSchedulerManager(self.conf, self.site_statistic,
self.seed_statistic)
self.selector = Selector(self.scheduler, self.conf)
self.crawl_selector = CrawlSelector(self.log, conf['selector_conf'],
conf['beanstalk_conf'],
self.scheduler)
def add_selector(self, title, values, event, *args, **kwargs):
"""
Agrega un selector (menú de opciones lateral) como entrada al menú.
:param title: Titulo del selector
:param values: Valores (entradas) del selector
:param event: Evento al seleccionar
:param args: Argumento de los eventos
:param kwargs: Parametros de los eventos
:return: void
"""
if kwargs.get("index") is not None:
self.actual.opciones.insert(
kwargs.get("index"),
[SELECTOR, Selector(title, values, event, *args)])
else:
self.actual.opciones.append(
[SELECTOR, Selector(title, values, event, *args)])
self.actual.size += 1
if self.actual.size > 1:
self.actual.posOptionY += -self.actual.fontsize / 2 - self.actual.optiondy / 2
