def exctract_metadata(text, isbn, container=dict()):
soup = BeautifulSoup(response_txt, "html.parser")
Option(soup.find("span", {"id": "productTitle"}))\
.map(lambda item: item.text)\
.map(lambda item: setitem(container, "title", item))\
.get_or(None)
Option(soup.find("span", {"class": "author"}))\
.map(lambda item: item.find("a"))\
.map(lambda item: item.text)\
.map(lambda item: setitem(container, "authors", item))\
.get_or(None)
Option(soup.find("div", {"id": "bookDescription_feature_div"}))\
.map(lambda item: item.find("noscript"))\
.map(lambda item: item.find("div"))\
.map(lambda item: item.text)\
.map(lambda item: setitem(container, "description", item))\
.get_or(None)
Option(soup.find("div", {"id": "img-canvas"}))\
.map(lambda item: item.find("img"))\
.map(lambda item: item.get("data-a-dynamic-image", None))\
.map(lambda item: item.split("\""))\
.map(lambda item: item[1])\
.map(lambda item: setitem(container, "imageLinks", item))\
.get_or(None)
container["industryIdentifiers"] = isbn
container["source"] = "AM"
return container
def download_documents_list(self, topic_filter, folder):
filepath = os.path.join(folder, "".join([uuid1().hex, ".json"]))
self.download_file(self.amphora_api_url, filepath)
files = OrderedDict()
attributes = defaultdict(int)
setters = {
"Items.item.topic_id.number": lambda attributes, value: setitem(attributes, "topic_id", value),
"Items.item.item_id.number": lambda attributes, value: setitem(attributes, "item_id", value),
"Items.item.item_file_id.number": lambda attributes, value: setitem(attributes, "file_id", value),
"Items.item.public_title.string": lambda attributes, value: setitem(attributes, "title", value),
"Items.item.object_date.string": lambda attributes, value: setitem(attributes, "date", value),
"Items.item.end_map": lambda attributes, value: attributes.clear()
}
with io.open(filepath, "r") as f:
parser = ijson_parse(f)
for prefix, event, value in parser:
if (prefix, event) == ("Items.item", "end_map"):
if attributes["topic_id"] in topic_filter and attributes["item_id"] and attributes["file_id"]:
files[attributes["item_id"]] = (attributes["file_id"],
attributes["topic_id"],
attributes["title"],
attributes["date"])
setter = setters.get("".join([prefix, ".", event]))
if setter:
setter(attributes, value)
return files
def __setitem__(self, key, value):
if isinstance(key, slice):
if hasattr(value, "__getitem__"):
operator.setitem(self._datap, key, value)
else:
raise TypeError("Value must be a sequence")
else:
self._datap[key] = self.dtype(value)
def mutate(self, key, index, value):
target = self._get_mutation_target(key)
if isinstance(index, tuple):
if isinstance(index[0], tuple):
index = tuple(slice(*e) for e in index)
else:
index = slice(*index)
setitem(target, index, value)
def set_text_entry(field, output):
txt = field.get_string_value().encode('utf-8').strip()
if len(txt) == 0:
operator.setitem(output, 'value', None)
elif len(txt) == 1:
operator.setitem(output, 'value', txt)
else:
field.set_value("")
mforms.Utilities.show_error(self.main.title, "Due to the nature of this wizard, you can't use unicode characters in this place, as also only one character is allowed.","Ok","","")
def codeMod_Start(self, name, attrs):
log(1, "Fun "+name+"_Start", attrs)
'''re-init local variables and the testcases for
the code module'''
if 'func' in attrs:
self.elements['func']=attrs['func']
else:
self.elements['func']=None
operator.setitem(self.elements, 'lvariable', '\n')
operator.setitem(self.elements, 'testCase', '\n')
def setitem(self, w_obj, w_key, w_val):
if self.concrete_mode:
try:
obj = self.unwrap_for_computation(w_obj)
key = self.unwrap_for_computation(w_key)
val = self.unwrap_for_computation(w_val)
operator.setitem(obj, key, val)
return self.w_None
except UnwrapException:
pass
return self.do_operation_with_implicit_exceptions("setitem", w_obj, w_key, w_val)
def test_operations(self):
repl_cfg = {'members': [{}, {}]}
repl = ReplicaSet(repl_cfg)
self.assertEqual(len(self.rs), 0)
operator.setitem(self.rs, 1, repl)
self.assertEqual(len(self.rs), 1)
self.assertEqual(operator.getitem(self.rs, 1).repl_id, repl.repl_id)
operator.delitem(self.rs, 1)
self.assertEqual(len(self.rs), 0)
self.assertRaises(KeyError, operator.getitem, self.rs, 1)
def set_text_entry(field, output):
txt = field.get_string_value().encode('utf-8').strip()
if len(txt) == 0:
operator.setitem(output, 'value', None)
mforms.Utilities.add_timeout(0.1, self.call_create_preview_table)
elif len(txt) == 1:
operator.setitem(output, 'value', txt)
mforms.Utilities.add_timeout(0.1, self.call_create_preview_table)
else:
field.set_value("")
mforms.Utilities.show_error("Import Wizard", "Due to the nature of this wizard, you can't use unicode characters in this place, as only one character is allowed.","Ok","","")
def test_operations(self):
config = {'members': [{}, {}, {}]}
shard = Shard(config)
self.assertEqual(len(self.sh), 0)
operator.setitem(self.sh, 1, shard)
self.assertEqual(len(self.sh), 1)
self.assertEqual(operator.getitem(self.sh, 1)['id'], shard.id)
operator.delitem(self.sh, 1)
self.assertEqual(len(self.sh), 0)
self.assertRaises(KeyError, operator.getitem, self.sh, 1)
shard.cleanup()
def test_operations(self):
config = {'shards': [{}, {}, {}]}
cluster = ShardedCluster(config)
self.assertEqual(len(self.sh), 0)
operator.setitem(self.sh, 1, cluster)
self.assertEqual(len(self.sh), 1)
self.assertEqual(operator.getitem(self.sh, 1)['id'], cluster.id)
operator.delitem(self.sh, 1)
self.assertEqual(len(self.sh), 0)
self.assertRaises(KeyError, operator.getitem, self.sh, 1)
cluster.cleanup()
def __setitem__ (self,key,value):
if isinstance(key,slice):
if hasattr(value,'__getitem__'):
try:
operator.setitem(self._datap,key,value) # works for 2.6+
except TypeError:
operator.setslice(self._datap,key.start,key.stop, value) # deprecated in 2.6, works in 2.5
#self._datap.__setitem__(key,value)
else:
raise TypeError("Value must be a sequence")
else:
self._datap[key] = self.dtype(value)
def test_setitem_mask_broadcast(self, data, setter):
ser = pd.Series(data)
mask = np.zeros(len(data), dtype=bool)
mask[:2] = True
if setter: # loc
target = getattr(ser, setter)
else: # __setitem__
target = ser
operator.setitem(target, mask, data[10])
assert ser[0] == data[10]
assert ser[1] == data[10]
def mutate(self, key, index, value):
target = None
if key in self.local:
target = self.local[key]
if key in self.broadcast.read:
target = self.broadcast[key][1]
if target is None:
raise KeyError("Cannot mutate non-existing dataset")
if isinstance(index, tuple):
if isinstance(index[0], tuple):
index = tuple(slice(*e) for e in index)
else:
index = slice(*index)
setitem(target, index, value)
def show_histogram(evaluation,df_train):
'''
Produce an image of the histogram of tfidf
'''
### id_to_ev: rowid -> EVAL
id_to_ev = {}
evaluation.apply(
lambda row: operator.setitem(id_to_ev,row['rowid'],row['EVAL']),
axis=1)
### converts df['rowid'] into corresponding labels
ev_list = []
for i in range(df_train.shape[0]):
ev_list.append(id_to_ev[df_train.ix[i,'rowid']])
ev_list = pd.Series(ev_list)
### histogram of tfidf
plt.figure(figsize=(400/96,400/96),dpi=96)
plt.hist(
[df_train['tfidf'][ev_list == k] for k in [-1,1]],
stacked=True,
label=['-1','+1'],
color=['r','b'])
plt.legend()
plt.title("Histogram of tfidf")
plt.tight_layout()
plt.savefig('histogram-of-tfidf.png',dpi=96)
def test_respond(self):
test_box = box.Box(db_fn="test.db")
test_box.db.open_db()
# test_box.db = berkeley_db.BerkeleyDB("test.db")
out = [""]
ex_monster = monster.Monster("Smoke", "fluff", "normal", [])
exp = json.dumps(ex_monster.export_monster())
exp_enc = json.dumps(ex_monster.export_monster(pwd=">install gentoo"))
cases = (
("/uploadmonster %s" % exp, "200 OK"),
("/uploadmonster %s" % exp_enc, "200 OK"),
("hello world", "403 BAD_COMMAND hello"),
("HELLO WORLD", "403 BAD_COMMAND hello"),
("/echo hello world", "200 OK hello world"))
# Override the print_stream method so the test can use it.
# Uses operator.setitem to get around inability to use assignments.
test_box.print_stream = lambda y, z: operator.setitem(out, 0, z)
# test_box.respond_to_input(None, "hello world")
for sent, response in cases:
# print(sent)
test_box.respond_to_input(None, sent)
self.assertEqual(out[0].strip(), response)
# test_box.db.close()
os.remove("test.db")
def te_chat(work, slot, colour, text):
no_echo = [False]
echo_lines = []
event_type = None
agent = work.terraria.user
if slot == 255:
match = re.match(r'((?P<sn>\[\S+\])|\*(?P<an>\S+))\s*(?P<m>.*)', text)
if match:
event_type = 'MESSAGE' if match.group('sn') else 'ACTION'
event_name = match.group('sn') or match.group('an')
event_text = match.group('m')
else:
event_type = 'MESSAGE'
event_name = work.terraria_protocol.players.get(slot, slot)
event_text = text
if event_type is not None:
reply = bridge.substitute_reply(
context = work.terraria.name,
local_name = event_name,
msg_local = lambda m: terraria_protocol.chat(work, strip_codes(m)),
msg_bridge = lambda m: echo_lines.append('<%s> %s' % (agent, m)),
cancel_bridge = lambda: operator.setitem(no_echo, 0, True))
yield util.msign(ab_mode, ('BRIDGE', event_type), ab_mode,
event_name, work.terraria.name, event_text, reply)
if not no_echo[0] and slot != work.terraria_protocol.slot:
if slot != 255:
text = '<%s> %s' % (name, text)
echo_lines.insert(0, text)
for line in echo_lines:
line = bridge.substitute_text(work.terraria.name, line)
yield sign('TERRARIA', work, line)
def replaceOutputCode(output_code, used, old, new):
print("replace", used, old, new)
old_code = list(map(str, output_code))
o, a, b = output_code, used.begin, used.end
value = [(o[a], [1]), (o[b], [2,3])] + [(v, [1,2,3]) for v in o[a+1:b]]
[setitem(t, i, t[i].replace(old, new)) for t, v in value for i in v if t[i]]
[print(f"{line}\t", old_code[line], "->", code) for line, code in enumerate(output_code) if str(code) != old_code[line]]
def patchdict(k, v):
if k in os.environ:
self._cleanups.append(lambda old=os.environ[k]: setitem(os.environ, k, old))
else:
self._cleanups.append(lambda: delitem(os.environ, k))
os.environ[k] = v
def test_simple(self):
init = range(15, 25)
for fmt in formats:
alen = len(init)
int_array = ARRAY(fmt, alen)
ia = int_array(*init)
self.assertEqual(len(ia), alen)
values = [ ia[i] for i in range(len(init)) ]
self.assertEqual(values, init)
from operator import setitem
new_values = range(42, 42 + alen)
[ setitem(ia, n, new_values[n]) for n in range(alen) ]
values = [ ia[i] for i in range(len(init)) ]
self.assertEqual(values, new_values)
ia = int_array()
values = [ ia[i] for i in range(len(init)) ]
self.assertEqual(values, [0] * len(init))
self.assertRaises(IndexError, int_array, *range(alen * 2))
CharArray = ARRAY(c_char, 3)
ca = CharArray('a', 'b', 'c')
self.assertRaises(TypeError, CharArray, 'abc')
self.assertEqual(ca[0], 'a')
self.assertEqual(ca[1], 'b')
self.assertEqual(ca[2], 'c')
self.assertEqual(ca[-3], 'a')
self.assertEqual(ca[-2], 'b')
self.assertEqual(ca[-1], 'c')
self.assertEqual(len(ca), 3)
from operator import getslice, delitem
self.assertRaises(TypeError, getslice, ca, 0, 1, -1)
self.assertRaises(TypeError, delitem, ca, 0)
def test_setitem(self):
a = range(3)
self.failUnlessRaises(TypeError, operator.setitem, a)
self.failUnlessRaises(TypeError, operator.setitem, a, None, None)
self.failUnless(operator.setitem(a, 0, 2) is None)
self.assert_(a == [2, 1, 2])
self.assertRaises(IndexError, operator.setitem, a, 4, 2)
def test_simple(self):
# create classes holding simple numeric types, and check
# various properties.
init = range(15, 25)
for fmt in formats:
alen = len(init)
int_array = ARRAY(fmt, alen)
ia = int_array(*init)
# length of instance ok?
self.assertEqual(len(ia), alen)
# slot values ok?
values = [ia[i] for i in range(alen)]
self.assertEqual(values, init)
# out-of-bounds accesses should be caught
with self.assertRaises(IndexError): ia[alen]
with self.assertRaises(IndexError): ia[-alen-1]
# change the items
from operator import setitem
new_values = range(42, 42+alen)
[setitem(ia, n, new_values[n]) for n in range(alen)]
values = [ia[i] for i in range(alen)]
self.assertEqual(values, new_values)
# are the items initialized to 0?
ia = int_array()
values = [ia[i] for i in range(alen)]
self.assertEqual(values, [0] * alen)
# Too many initializers should be caught
self.assertRaises(IndexError, int_array, *range(alen*2))
CharArray = ARRAY(c_char, 3)
ca = CharArray("a", "b", "c")
# Should this work? It doesn't:
# CharArray("abc")
self.assertRaises(TypeError, CharArray, "abc")
self.assertEqual(ca[0], "a")
self.assertEqual(ca[1], "b")
self.assertEqual(ca[2], "c")
self.assertEqual(ca[-3], "a")
self.assertEqual(ca[-2], "b")
self.assertEqual(ca[-1], "c")
self.assertEqual(len(ca), 3)
# slicing is now supported, but not extended slicing (3-argument)!
from operator import getslice, delitem
self.assertRaises(TypeError, getslice, ca, 0, 1, -1)
# cannot delete items
self.assertRaises(TypeError, delitem, ca, 0)
def __enter__(self):
self.old_contexts = _state.contexts
self.new_contexts = (self.old_contexts +
((ExceptionStackContext, self.exception_handler,
self.active_cell),))
_state.contexts = self.new_contexts
return lambda: operator.setitem(self.active_cell, 0, False)
def test_setitem(self):
a = list(range(3))
self.assertRaises(TypeError, operator.setitem, a)
self.assertRaises(TypeError, operator.setitem, a, None, None)
self.assertTrue(operator.setitem(a, 0, 2) is None)
self.assertTrue(a == [2, 1, 2])
self.assertRaises(IndexError, operator.setitem, a, 4, 2)
def printOverallSuiteResults(self, r):
self.stream.writeln()
self.stream.writeln(self.separator4)
self.stream.writeln('OVERALL SUITE RESULTS')
fmt = [
('SUITE', 'suite', 50, 'left'),
('CASES', 'cases', 15, 'right'),
('PASSED', 'passed', 15, 'right'),
('FAILED', 'failed', 15, 'right'),
('SKIPPED', 'skipped', 15, 'right'),
('%', 'percentage', 20, 'right'),
('TIME (s)', 'time', 20, 'right')
]
data = []
map(lambda x: data.append({'suite': r[x]['name'],
'cases': r[x]['num_passed'] + r[x]['num_failed'],
'passed': r[x]['num_passed'],
'failed': r[x]['num_failed'],
'skipped': r[x]['num_skipped'],
'percentage': float(r[x]['num_passed'])/(r[x]['num_passed'] + r[x]['num_failed']) * 100 if (r[x]['num_passed'] + r[x]['num_failed']) > 0 else 0,
'time': r[x]['execution_time']}), r)
total_suites_passed = len([x for x in data if not x['failed']])
total_suites_passed_percentage = format(float(total_suites_passed)/len(data) * 100, '.%sf' %CustomTextTestResult._pass_percentage_significant_digits)
totals = {'suite': 'TOTALS %s/%s (%s%%) suites passed' %(total_suites_passed, len(data), total_suites_passed_percentage),
'cases': sum_dict_key(data, 'cases'),
'passed': sum_dict_key(data, 'passed'),
'failed': sum_dict_key(data, 'failed'),
'skipped': sum_dict_key(data, 'skipped'),
'percentage': sum_dict_key(data, 'percentage')/len(data),
'time': sum_dict_key(data, 'time', 'float')}
map(lambda x: operator.setitem(x, 'percentage', format(x['percentage'], '.%sf' %CustomTextTestResult._pass_percentage_significant_digits)), data)
totals['percentage'] = format(totals['percentage'], '.%sf' %CustomTextTestResult._pass_percentage_significant_digits)
self.stream.writeln( TablePrinter(fmt, tl=self.separator1, ul=self.separator2, bl=self.separator3)(data, totals) )
self.stream.writeln()
def parse_cas_tree(tree):
"""Pull data out of ElementTree ```tree`` into a dict.
This maps the CAS attributes to normalized names using
:const:`CAS_ATTR_MAP`.
``tree`` is an :class:`xml.etree.elementtree.Element` created from
the XML response returned from the PSU CAS system. Its structure
looks like this::
<cas:serviceResponse xmlns:cas='http://www.yale.edu/tp/cas'>
<cas:authenticationSuccess>
<cas:user>bobs</cas:user>
<cas:attributes>
<cas:uid>bobs</cas:uid>
<cas:display_name>Bob Smith</cas:display_name>
<cas:given_name>Bob</cas:given_name>
<cas:sn>Smith</cas:sn>
<cas:mail>[email protected]</cas:mail>
</cas:attributes>
</cas:authenticationSuccess>
</cas:serviceResponse>
"""
success = tree_find(tree, 'cas:authenticationSuccess')
attributes = tree_find(success, 'cas:attributes')
cas_data = {}
for name, cas_name in CAS_ATTR_MAP:
with get_cas_attr(attributes, cas_name) as option:
option(some=lambda v: setitem(cas_data, name, v))
return cas_data
def mutate(matrix, vlist, vindex):
"""mutate(matrix_list[], vowel_list[], int(vowel_list)) matrix list is the
entire result set, calculated by raising 5 (the count of all vowels) to
the power of x (the number of vowels in the name). For example the name
Mutuma has 3 vowels namely "u", "u" and "a".
"""
for position in vindex:
matrix.sort()
for i in range(len(matrix)):
name_list = list(matrix[i])
setitem(name_list, position, vlist[i])
temp_string = ""
for letter in name_list:
temp_string += letter
matrix[i] = temp_string
return matrix
def get_component_comments(request):
# try:
gcomponent = GComponents.objects.get(pk = int(request.GET['pk']))
comments = Comment.objects.all().filter(component = gcomponent).annotate(rank=Sum('cranking__rank')).order_by("rank", "-date")
dict_comments = map(model_to_dict, comments)
map(lambda x: operator.setitem(x, 'date', str(x['date'])), dict_comments)
map(lambda x: operator.setitem(x, 'commenter', User_Profiles.objects.get(user = Users.objects.get(pk = int(x['commenter']))).username), dict_comments)
map(lambda x: operator.setitem(x, 'rank', Comment.objects.all().filter(pk = x['id']).aggregate(rank=Sum('cranking__rank'))['rank']), dict_comments)
map(lambda x: operator.setitem(x, 'rank', 0 if x['rank'] is None else x['rank']), dict_comments)
response = HttpResponse(json.dumps(dict_comments))
response.status_code = 200
return response
def load_file(self, f, setting=None):
if setting is None:
setting = settings()
setting.set(setting.USE_PYTHON_OPENCASCADE, True)
v = self._display
t = {0: time.time()}
def update(dt = None):
t1 = time.time()
if t1 - t[0] > (dt or -1):
v.FitAll()
v.Repaint()
t[0] = t1
terminate = [False]
self.window.window_closed.connect(lambda *args: operator.setitem(terminate, 0, True))
for p in f.by_type("IfcProduct"):
if terminate[0]: break
if p.Representation is None: continue
shape = create_shape(setting, p)
ais = display_shape(shape, viewer_handle=v)
ais.GetObject().SetSelectionPriority(self.counter)
self.ais_to_product[self.counter] = p
self.product_to_ais[p] = ais
self.counter += 1
QtGui.QApplication.processEvents()
if p.is_a() in {'IfcSpace', 'IfcOpeningElement'}:
v.Context.Erase(ais, True)
update(0.1)
update()
def input_sensitivity(self, summarize=True):
if summarize:
return reduce(np.add, [k.input_sensitivity(summarize) for k in self.parts])
else:
i_s = np.zeros((len(self.parts), self.input_dim))
from operator import setitem
[setitem(i_s, (i, Ellipsis), k.input_sensitivity(summarize)) for i, k in enumerate(self.parts)]
return i_s
def set_selector_entry(selector, output):
operator.setitem(
output, 'value',
output['opts'][str(selector.get_string_value())])
def __enter__(self):
self.old_contexts = _state.contexts
_state.contexts = (self.old_contexts +
((ExceptionStackContext, self.exception_handler,
self.active_cell), ))
return lambda: operator.setitem(self.active_cell, 0, False)
def forward(self, xs, lengths, t):
max_length, batch_size = xs.size()
h = Variable(xs.data.new(1, batch_size,
self.hidden_size).zero_().float(),
requires_grad=False)
state = (h, h)
embs = functional.dropout(self.embed(xs), 0.1)
rows = xs.data.new(batch_size).zero_()
columns = xs.data.new(range(batch_size))
log_probs = []
baselines = []
masks = []
hiddens = [None] * batch_size
last_rows = rows.clone().fill_(max_length - 1)
for _ in range(self.N):
for _ in range(self.R):
feed_previous = rows >= max_length
rows = feed_previous.long() * last_rows + (
1 - feed_previous.long()) * rows
emb = embs[rows, columns]
if feed_previous.any():
[
setitem(hiddens, i, h[:, i, :])
for i, v in enumerate(feed_previous) if v == 1
]
h, state = self.lstm(emb[None], state)
rows = rows + 1
if self._finish_reading(rows, max_length):
break
feed_previous = rows >= max_length
# TODO: replace where function when it is added
rows = feed_previous.long() * last_rows + (
1 - feed_previous.long()) * rows
if feed_previous.any():
[
setitem(hiddens, i, h[:, i, :])
for i, v in enumerate(feed_previous) if v == 1
]
h, state = self.lstm(embs[rows, columns][None], state)
p = functional.softmax(self.linear(h.squeeze(0)), dim=1)
m = Categorical(p)
jump = m.sample()
log_prob = m.log_prob(jump)
log_probs.append(log_prob[:, None])
masks.append(feed_previous[:, None])
baselines.append(self.baseline(h.squeeze(0)))
is_stopping = (jump.data == 0).long()
rows = is_stopping * (last_rows +
1) + (1 - is_stopping) * (rows + jump.data)
if self._finish_reading(rows, max_length):
break
if any(x is None for x in hiddens):
[
setitem(hiddens, i, h[:, i, :]) for i, v in enumerate(hiddens)
if v is None
]
h = torch.cat(hiddens, dim=0)
y = functional.log_softmax(self.output(h), dim=1)
log_prob = torch.cat(log_probs, dim=1)
baseline = torch.cat(baselines, dim=1)
reward = self._get_reward(y, t).expand_as(baseline)
# filling with 0
mask = torch.cat(masks, dim=1)
log_prob.data.masked_fill_(mask, 0)
baseline.data.masked_fill_(mask, 0)
reward.data.masked_fill_(mask, 0)
return self.nll_loss(y, t) + self._reinforce(log_prob, reward, baseline) + \
self.mse_loss(baseline, reward)
print(operator.add(99, 1)) #addition
print(list(map(operator.neg, range(9)))) #negation
print(operator.truediv(3, 2))
print(operator.floordiv(3, 2))
l = [1, 2, 3, 4]
print(functools.reduce(operator.mul,
l)) #reduced than reduce(lambda x,y:x*y,l)
print(operator.lt(2, 9)) #less than
print(operator.is_(1, 10)) #is_ to remove conflict from is builtin
'getters and setters'
my_list = [1, 2, 3, 5]
print(operator.getitem(my_list, 2)) #equivalent to my_list[2]
operator.setitem(my_list, 2, 222) #appending
operator.delitem(my_list, 1)
print(my_list)
'itemgetter acts as a partial function'
f = operator.itemgetter(2) #returns a function
print(type(f))
print(f(my_list)) #->f=operator.getitem(mylist,2)
print(f('Jenish'))
'can have more than one argument'
f = operator.itemgetter(2, 3, 4) #returns a function
print(f('the quick brown fox jumped over the lazy brown fox'))
def class_(
class_def,
class_name=None,
merge_inner_function=None,
infer_type=False,
word_wrap=True,
):
"""
Converts an AST to our IR
:param class_def: Class AST or Module AST with a ClassDef inside
:type class_def: ```Union[Module, ClassDef]```
:param class_name: Name of `class`. If None, gives first found.
:type class_name: ```Optional[str]```
:param merge_inner_function: Name of inner function to merge. If None, merge nothing.
:type merge_inner_function: ```Optional[str]```
:param infer_type: Whether to try inferring the typ (from the default)
:type infer_type: ```bool```
:param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length.
:type word_wrap: ```bool```
:returns: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:rtype: ```dict```
"""
assert not isinstance(class_def, FunctionDef)
is_supported_ast_node = isinstance(class_def, (Module, ClassDef))
if not is_supported_ast_node and isinstance(class_def, type):
ir = _inspect(class_def, class_name, word_wrap)
parsed_body = ast.parse(getsource(class_def).lstrip()).body[0]
parsed_body.body = (parsed_body.body
if ast.get_docstring(parsed_body) is None else
parsed_body.body[1:])
if merge_inner_function is not None:
_merge_inner_function(
parsed_body,
infer_type=infer_type,
intermediate_repr=ir,
merge_inner_function=merge_inner_function,
)
return ir
ir["_internal"] = {
"body":
list(
filterfalse(
rpartial(isinstance, AnnAssign),
parsed_body.body,
)),
"from_name":
class_name,
"from_type":
"cls",
}
body_ir = class_(
class_def=parsed_body,
class_name=class_name,
merge_inner_function=merge_inner_function,
)
ir_merge(ir, body_ir)
return ir
assert (is_supported_ast_node
), "Expected 'Union[Module, ClassDef]' got `{!r}`".format(
type(class_def).__name__)
class_def = find_ast_type(class_def, class_name)
doc_str = get_docstring(class_def)
intermediate_repr = ({
"name": class_name,
"type": "static",
"doc": "",
"params": OrderedDict(),
"returns": None,
} if doc_str is None else docstring(get_docstring(class_def).replace(
":cvar", ":param"),
emit_default_doc=False))
if "return_type" in intermediate_repr["params"]:
intermediate_repr["returns"] = OrderedDict(
(("return_type",
intermediate_repr["params"].pop("return_type")), ))
body = class_def.body if doc_str is None else class_def.body[1:]
for e in body:
if isinstance(e, AnnAssign):
typ = to_code(e.annotation).rstrip("\n")
val = (lambda v: {
"default": NoneStr
} if v is None else {
"default":
v if type(v).__name__ in simple_types else (lambda value: {
"{}": {} if isinstance(v, Dict) else set(),
"[]": [],
"()": (),
}.get(value, parse_to_scalar(value)))(to_code(v).rstrip("\n"))
})(get_value(get_value(e)))
# if 'str' in typ and val: val["default"] = val["default"].strip("'") # Unquote?
typ_default = dict(typ=typ, **val)
for key in "params", "returns":
if e.target.id in (intermediate_repr[key] or iter(())):
intermediate_repr[key][e.target.id].update(typ_default)
typ_default = False
break
if typ_default:
k = "returns" if e.target.id == "return_type" else "params"
if intermediate_repr.get(k) is None:
intermediate_repr[k] = OrderedDict()
intermediate_repr[k][e.target.id] = typ_default
elif isinstance(e, Assign):
val = get_value(e)
if val is not None:
val = get_value(val)
deque(
map(
lambda target: setitem(*(
(intermediate_repr["params"][target.id], "default",
val)
if target.id in intermediate_repr["params"] else (
intermediate_repr["params"],
target.id,
{
"default": val
},
))),
e.targets,
),
maxlen=0,
)
intermediate_repr.update({
"params":
OrderedDict(
map(
partial(_set_name_and_type,
infer_type=infer_type,
word_wrap=word_wrap),
intermediate_repr["params"].items(),
)),
"_internal": {
"body":
list(filterfalse(rpartial(isinstance, (AnnAssign, Assign)), body)),
"from_name":
class_def.name,
"from_type":
"cls",
},
})
if merge_inner_function is not None:
assert isinstance(class_def, ClassDef)
_merge_inner_function(
class_def,
infer_type=infer_type,
intermediate_repr=intermediate_repr,
merge_inner_function=merge_inner_function,
)
# intermediate_repr['_internal']["body"]= list(filterfalse(rpartial(isinstance,(AnnAssign,Assign)),class_def.body))
return intermediate_repr
import operator
# Set Item , Delete Item, Get Item
li = [1, 2, 3, 4]
print(li)
operator.setitem(li, 3, 8)
print(li)
d = operator.getitem(li, 3)
print(d)
operator.delitem(li, 0)
print(li)
lis = [1, 2, 3, 4, 5]
print(lis)
operator.setitem(lis, slice(0, 2), [5, 6, 7])
print(lis)
s = operator.getitem(lis, slice(0, 4))
print(s)
operator.delitem(lis, slice(0, 4))
print(lis)
s1 = "GeekforGeeks"
s2 = "Communication"
print(operator.concat(s1, s2))
if operator.contains(s1, s2):
print("It contains Geekforgeeks")
else:
def __new__(cls, name, bases, namespace, *, decorated=None, **kwds):
"""
Updates attributes in the namespace of the new class
using the decorators and order specified in 'decorated'.
"""
# The mro will be needed for the resolution of inherited attributes.
new_cls = type.__new__(cls, name, bases, namespace, **kwds)
# Subclasses of an instance might not want to decorated anything.
if decorated is None:
return new_cls
if not isinstance(decorated, dict):
raise TypeError("'decorated' must be a dictionary.")
error = 0
errors = (
None,
TypeError("The values in 'decorated' must contain strings only."),
TypeError("The decorators must be callable."),
TypeError("All values in 'decorated' must be iterable."),
)
# It's **less costly** to ask for forgiveness than permission
# since any program using this shouldn't even proceed
# in the case of any of these errors.
# Also, it's safe to raise errors after partial modification
# of the namespace since the class won't be created
# and in a case where the metaclass is explicity called,
# the namespace dict normally shouldn't be used for anything else.
try:
for deco, attrs in decorated.items():
for attr in attrs:
if not isinstance(attr, str):
error = 1
return # A `break` is not sufficient in a nested loop.
try:
# Standard Attribute Resolution (with possible repetition)
for _cls in new_cls.__mro__ + cls.__mro__:
if attr in _cls.__dict__:
obj = _cls.__dict__.get(attr)
setitem(
namespace,
attr,
(classmethod(deco(obj.__func__))
if isinstance(obj, classmethod) else
staticmethod(deco(obj.__func__))
if isinstance(
obj, staticmethod) else deco(obj)),
)
break
else:
raise AttributeError(
"The class has no attribute %r." % attr)
except TypeError as err:
print(err)
error = 2
return # A `break` is not sufficient in a nested loop.
except TypeError:
error = 3
finally:
if error:
raise errors[error]
return type.__new__(cls, name, bases, namespace, **kwds)
def _get_or_create(d, k):
# TODO (max) may handle to large array idx (add elm)
if not isinstance(d, list) and not operator.contains(d, k):
operator.setitem(d, k, {})
return operator.getitem(d, k)
def _setitem(obj, attr, val):
return operator.setitem(obj, int(attr), val)
def test_setitem_scalar(self, data, setter):
arr = pd.Series(data)
setter = getattr(arr, setter)
operator.setitem(setter, 0, data[1])
assert arr[0] == data[1]
# [(1, 2, 3), (2, 3, 4), (3, 4), (4,)]
# Slicing
l = [1, 2, 3, 4]
l[0:2]
l[0:2] = ['a', 'b', 'c']
print(l)
# ['a', 'b', 'c', 3, 4]
del l[3:5]
print(l)
# ['a', 'b', 'c']
# Using operator
l = [1, 2, 3, 4]
operator.getitem(l, slice(0, 2))
# [1, 2]
operator.setitem(l, slice(0, 2), ['a', 'b', 'c'])
print(l)
# ['a', 'b', 'c', 3, 4]
operator.delitem(l, slice(3, 5))
print(l)
# ['a', 'b', 'c']
# Calling another Callable
x = 'python'
x.upper()
operator.methodcaller('upper')('python')
# 'PYTHON'
operator.attrgetter('upper')(x)()
# 'PYTHON'
class TestWebKitElement:
"""Generic tests for WebKitElement.
Note: For some methods, there's a dedicated test class with more involved
tests.
"""
@pytest.fixture
def elem(self):
return get_webelem()
def test_nullelem(self):
"""Test __init__ with a null element."""
with pytest.raises(webkitelem.IsNullError):
get_webelem(null=True)
def test_double_wrap(self, elem):
"""Test wrapping a WebKitElement."""
with pytest.raises(TypeError, match="Trying to wrap a wrapper!"):
webkitelem.WebKitElement(elem, tab=None)
@pytest.mark.parametrize('code', [
pytest.param(str, id='str'),
pytest.param(lambda e: e[None], id='getitem'),
pytest.param(lambda e: operator.setitem(e, None, None), id='setitem'),
pytest.param(lambda e: operator.delitem(e, None), id='delitem'),
pytest.param(lambda e: None in e, id='contains'),
pytest.param(list, id='iter'),
pytest.param(len, id='len'),
pytest.param(lambda e: e.has_frame(), id='has_frame'),
pytest.param(lambda e: e.geometry(), id='geometry'),
pytest.param(lambda e: e.value(), id='value'),
pytest.param(lambda e: e.set_value('foo'), id='set_value'),
pytest.param(lambda e: e.insert_text('foo'), id='insert_text'),
pytest.param(lambda e: e.is_writable(), id='is_writable'),
pytest.param(lambda e: e.is_content_editable(),
id='is_content_editable'),
pytest.param(lambda e: e.is_editable(), id='is_editable'),
pytest.param(lambda e: e.is_text_input(), id='is_text_input'),
pytest.param(lambda e: e.remove_blank_target(),
id='remove_blank_target'),
pytest.param(lambda e: e.outer_xml(), id='outer_xml'),
pytest.param(lambda e: e.tag_name(), id='tag_name'),
pytest.param(lambda e: e.rect_on_view(), id='rect_on_view'),
pytest.param(lambda e: e._is_visible(None), id='is_visible'),
])
def test_vanished(self, elem, code):
"""Make sure methods check if the element is vanished."""
elem._elem.isNull.return_value = True
elem._elem.tagName.return_value = 'span'
with pytest.raises(webkitelem.IsNullError):
code(elem)
def test_str(self, elem):
assert str(elem) == 'text'
wke_qualname = 'qutebrowser.browser.webkit.webkitelem.WebKitElement'
@pytest.mark.parametrize('is_null, xml, expected', [
(False, '<fakeelem/>', "<{} html='<fakeelem/>'>".format(wke_qualname)),
(False, '<foo>\n<bar/>\n</foo>',
"<{} html='<foo><bar/></foo>'>".format(wke_qualname)),
(False, '<foo>{}</foo>'.format('x' * 500),
"<{} html='<foo>{}…'>".format(wke_qualname, 'x' * 494)),
(True, None, '<{} html=None>'.format(wke_qualname)),
])
def test_repr(self, elem, is_null, xml, expected):
elem._elem.isNull.return_value = is_null
elem._elem.toOuterXml.return_value = xml
assert repr(elem) == expected
def test_getitem(self):
elem = get_webelem(attributes={'foo': 'bar'})
assert elem['foo'] == 'bar'
def test_getitem_keyerror(self, elem):
with pytest.raises(KeyError):
elem['foo'] # pylint: disable=pointless-statement
def test_setitem(self, elem):
elem['foo'] = 'bar'
assert elem._elem.attribute('foo') == 'bar'
def test_delitem(self):
elem = get_webelem(attributes={'foo': 'bar'})
del elem['foo']
assert not elem._elem.hasAttribute('foo')
def test_setitem_keyerror(self, elem):
with pytest.raises(KeyError):
del elem['foo']
def test_contains(self):
elem = get_webelem(attributes={'foo': 'bar'})
assert 'foo' in elem
assert 'bar' not in elem
def test_not_eq(self):
one = get_webelem()
two = get_webelem()
assert one != two
def test_eq(self):
one = get_webelem()
two = webkitelem.WebKitElement(one._elem, tab=None)
assert one == two
def test_eq_other_type(self):
assert get_webelem() != object()
@pytest.mark.parametrize('attributes, expected', [
({
'one': '1',
'two': '2'
}, {'one', 'two'}),
({}, set()),
])
def test_iter(self, attributes, expected):
elem = get_webelem(attributes=attributes)
assert set(elem) == expected
@pytest.mark.parametrize('attributes, length', [
({
'one': '1',
'two': '2'
}, 2),
({}, 0),
])
def test_len(self, attributes, length):
elem = get_webelem(attributes=attributes)
assert len(elem) == length
@pytest.mark.parametrize('attributes, writable', [
([], True),
(['disabled'], False),
(['readonly'], False),
(['disabled', 'readonly'], False),
])
def test_is_writable(self, attributes, writable):
elem = get_webelem(attributes=attributes)
assert elem.is_writable() == writable
@pytest.mark.parametrize('attributes, expected', [
({}, False),
({
'contenteditable': 'false'
}, False),
({
'contenteditable': 'inherit'
}, False),
({
'contenteditable': 'true'
}, True),
])
def test_is_content_editable(self, attributes, expected):
elem = get_webelem(attributes=attributes)
assert elem.is_content_editable() == expected
@pytest.mark.parametrize('tagname, attributes, expected', [
('input', {}, True),
('textarea', {}, True),
('select', {}, False),
('foo', {
'role': 'combobox'
}, True),
('foo', {
'role': 'textbox'
}, True),
('foo', {
'role': 'bar'
}, False),
('input', {
'role': 'bar'
}, True),
])
def test_is_text_input(self, tagname, attributes, expected):
elem = get_webelem(tagname=tagname, attributes=attributes)
assert elem.is_text_input() == expected
@pytest.mark.parametrize('attribute, code', [
('geometry', lambda e: e.geometry()),
('toOuterXml', lambda e: e.outer_xml()),
])
def test_simple_getters(self, elem, attribute, code):
sentinel = object()
mock = getattr(elem._elem, attribute)
setattr(mock, 'return_value', sentinel)
assert code(elem) is sentinel
@pytest.mark.parametrize('frame, expected', [(object(), True),
(None, False)])
def test_has_frame(self, elem, frame, expected):
elem._elem.webFrame.return_value = frame
assert elem.has_frame() == expected
def test_tag_name(self, elem):
elem._elem.tagName.return_value = 'SPAN'
assert elem.tag_name() == 'span'
def test_value(self, elem):
elem._elem.evaluateJavaScript.return_value = 'js'
assert elem.value() == 'js'
@pytest.mark.parametrize('editable, value, uses_js, arg', [
('false', 'foo', True, 'this.value="foo"'),
('false', "foo'bar", True, r'this.value="foo\'bar"'),
('true', 'foo', False, 'foo'),
])
def test_set_value(self, editable, value, uses_js, arg):
elem = get_webelem(attributes={'contenteditable': editable})
elem.set_value(value)
attr = 'evaluateJavaScript' if uses_js else 'setPlainText'
called_mock = getattr(elem._elem, attr)
called_mock.assert_called_with(arg)
import operator
my_list = [1, 2, 3, 4]
operator.setitem(my_list,1,100)
print(my_list)
operator.delitem(my_list,3)
print(my_list)
f = operator.itemgetter(2)
print(f(my_list))
## itemgetter return in tuple
s = "python"
print("String : ",s)
print(f(s))
### attribute getter
d = {"name":"vaneet","lastname":"gupta"}
sort = sorted(d.items(), key=operator.itemgetter(1))
print(sort)
def test_modify_tree(self):
tree = self.repo[TREE_SHA]
with pytest.raises(TypeError):
operator.setitem('c', tree['a'])
with pytest.raises(TypeError):
operator.delitem('c')
def test_simple(self):
# create classes holding simple numeric types, and check
# various properties.
init = range(15, 25)
for fmt in formats:
alen = len(init)
int_array = ARRAY(fmt, alen)
ia = int_array(*init)
# length of instance ok?
self.assertEqual(len(ia), alen)
# slot values ok?
values = [ia[i] for i in range(alen)]
self.assertEqual(values, init)
# out-of-bounds accesses should be caught
with self.assertRaises(IndexError):
ia[alen]
with self.assertRaises(IndexError):
ia[-alen - 1]
# change the items
from operator import setitem
new_values = range(42, 42 + alen)
[setitem(ia, n, new_values[n]) for n in range(alen)]
values = [ia[i] for i in range(alen)]
self.assertEqual(values, new_values)
# are the items initialized to 0?
ia = int_array()
values = [ia[i] for i in range(alen)]
self.assertEqual(values, [0] * alen)
# Too many initializers should be caught
self.assertRaises(IndexError, int_array, *range(alen * 2))
CharArray = ARRAY(c_char, 3)
ca = CharArray("a", "b", "c")
# Should this work? It doesn't:
# CharArray("abc")
self.assertRaises(TypeError, CharArray, "abc")
self.assertEqual(ca[0], "a")
self.assertEqual(ca[1], "b")
self.assertEqual(ca[2], "c")
self.assertEqual(ca[-3], "a")
self.assertEqual(ca[-2], "b")
self.assertEqual(ca[-1], "c")
self.assertEqual(len(ca), 3)
# slicing is now supported, but not extended slicing (3-argument)!
from operator import getslice, delitem
self.assertRaises(TypeError, getslice, ca, 0, 1, -1)
# cannot delete items
self.assertRaises(TypeError, delitem, ca, 0)
class Field(Metadata):
"""Field representation
API | Usage
-------- | --------
Public | `from frictionless import Field`
Parameters:
descriptor? (str|dict): field descriptor
name? (str): field name (for machines)
title? (str): field title (for humans)
descriptor? (str): field descriptor
type? (str): field type e.g. `string`
format? (str): field format e.g. `default`
missing_values? (str[]): missing values
constraints? (dict): constraints
rdf_type? (str): RDF type
schema? (Schema): parent schema object
Raises:
FrictionlessException: raise any error that occurs during the process
"""
def __init__(
self,
descriptor=None,
*,
# Spec
name=None,
title=None,
description=None,
type=None,
format=None,
missing_values=None,
constraints=None,
rdf_type=None,
# Extra
schema=None,
):
self.setinitial("name", name)
self.setinitial("title", title)
self.setinitial("description", description)
self.setinitial("type", type)
self.setinitial("format", format)
self.setinitial("missingValues", missing_values)
self.setinitial("constraints", constraints)
self.setinitial("rdfType", rdf_type)
self.__schema = schema
self.__type = None
super().__init__(descriptor)
# Replace deprecated "fmt:"
format = self.get("format")
if format and isinstance(format, str) and format.startswith("fmt:"):
message = 'Format "fmt:<PATTERN>" is deprecated. Please remove "fmt:" prefix.'
warnings.warn(message, UserWarning)
self["format"] = format.replace("fmt:", "")
@Metadata.property
def name(self):
"""
Returns:
str: name
"""
return self.get("name", self.type)
@Metadata.property
def title(self):
"""
Returns:
str?: title
"""
return self.get("title")
@Metadata.property
def description(self):
"""
Returns:
str?: description
"""
return self.get("description")
@Metadata.property
def type(self):
"""
Returns:
str: type
"""
return self.get("type", "any")
@Metadata.property
def format(self):
"""
Returns:
str: format
"""
format = self.get("format", "default")
return format
@Metadata.property
def missing_values(self):
"""
Returns:
str[]: missing values
"""
schema = self.__schema
default = schema.missing_values if schema else copy(config.DEFAULT_MISSING_VALUES)
missing_values = self.get("missingValues", default)
return self.metadata_attach("missingValues", missing_values)
@Metadata.property
def constraints(self):
"""
Returns:
dict: constraints
"""
constraints = self.get("constraints", {})
return self.metadata_attach("constraints", constraints)
@Metadata.property
def rdf_type(self):
"""
Returns:
str?: RDF Type
"""
return self.get("rdfType")
@Metadata.property(
write=lambda self, value: setitem(self.constraints, "required", value)
)
def required(self):
"""
Returns:
bool: if field is requried
"""
return self.constraints.get("required", False)
@property
def builtin(self):
"""
Returns:
bool: returns True is the type is not custom
"""
return self.__type.builtin
@property
def schema(self):
"""
Returns:
Schema?: parent schema
"""
return self.__schema
# Boolean
@Metadata.property
def true_values(self):
"""
Returns:
str[]: true values
"""
true_values = self.get("trueValues", config.DEFAULT_TRUE_VALUES)
return self.metadata_attach("trueValues", true_values)
@Metadata.property
def false_values(self):
"""
Returns:
str[]: false values
"""
false_values = self.get("falseValues", config.DEFAULT_FALSE_VALUES)
return self.metadata_attach("falseValues", false_values)
# Integer/Number
@Metadata.property
def bare_number(self):
"""
Returns:
bool: if a bare number
"""
return self.get("bareNumber", config.DEFAULT_BARE_NUMBER)
@Metadata.property
def float_number(self):
"""
Returns:
bool: whether it's a floating point number
"""
return self.get("floatNumber", config.DEFAULT_FLOAT_NUMBER)
@Metadata.property
def decimal_char(self):
"""
Returns:
str: decimal char
"""
return self.get("decimalChar", config.DEFAULT_DECIMAL_CHAR)
@Metadata.property
def group_char(self):
"""
Returns:
str: group char
"""
return self.get("groupChar", config.DEFAULT_GROUP_CHAR)
# Expand
def expand(self):
"""Expand metadata"""
self.setdefault("name", "field")
self.setdefault("type", "any")
self.setdefault("format", "default")
# Boolean
if self.type == "boolean":
self.setdefault("trueValues", self.true_values)
self.setdefault("falseValues", self.false_values)
# Integer/Number
if self.type in ["integer", "number"]:
self.setdefault("bareNumber", self.bare_number)
if self.type == "number":
self.setdefault("decimalChar", self.decimal_char)
self.setdefault("groupChar", self.group_char)
# Read
def read_cell(self, cell):
"""Read cell
Parameters:
cell (any): cell
Returns:
(any, OrderedDict): processed cell and dict of notes
"""
notes = None
if cell in self.missing_values:
cell = None
if cell is not None:
cell = self.__type.read_cell(cell)
if cell is None:
notes = notes or OrderedDict()
notes["type"] = f'type is "{self.type}/{self.format}"'
if not notes and self.read_cell_checks:
for name, check in self.read_cell_checks.items():
if not check(cell):
notes = notes or OrderedDict()
notes[name] = f'constraint "{name}" is "{self.constraints[name]}"'
return cell, notes
def read_cell_convert(self, cell):
"""Read cell (convert only)
Parameters:
cell (any): cell
Returns:
any/None: processed cell or None if an error
"""
return self.__type.read_cell(cell)
@Metadata.property(write=False)
def read_cell_checks(self):
"""Read cell (checks only)
Returns:
OrderedDict: dictionlary of check function by a constraint name
"""
checks = OrderedDict()
for name in self.__type.constraints:
constraint = self.constraints.get(name)
if constraint is not None:
if name in ["minimum", "maximum"]:
constraint = self.__type.read_cell(constraint)
if name == "enum":
constraint = list(map(self.__type.read_cell, constraint))
checks[name] = partial(globals().get(f"check_{name}"), constraint)
return checks
# Write
def write_cell(self, cell, *, ignore_missing=False):
"""Write cell
Parameters:
cell (any): cell to convert
ignore_missing? (bool): don't convert None values
Returns:
(any, OrderedDict): processed cell and dict of notes
"""
notes = None
if cell is None:
missing_value = cell if ignore_missing else self.write_cell_missing_value
return missing_value, notes
cell = self.__type.write_cell(cell)
if cell is None:
notes = notes or OrderedDict()
notes["type"] = f'type is "{self.type}/{self.format}"'
return cell, notes
def write_cell_convert(self, cell):
"""Write cell (convert only)
Parameters:
cell (any): cell
Returns:
any/None: processed cell or None if an error
"""
return self.__type.write_cell(cell)
@Metadata.property(write=False)
def write_cell_missing_value(self):
"""Write cell (missing value only)
Returns:
str: a value to replace None cells
"""
if self.missing_values:
return self.missing_values[0]
return config.DEFAULT_MISSING_VALUES[0]
# Metadata
def metadata_process(self):
# Type
try:
self.__type = system.create_type(self)
except FrictionlessException:
self.__type = types.AnyType(self)
def metadata_validate(self):
yield from super().metadata_validate()
# Constraints
for name in self.constraints.keys():
if name not in self.__type.constraints + ["unique"]:
note = f'constraint "{name}" is not supported by type "{self.type}"'
yield errors.SchemaError(note=note)
# Metadata
metadata_Error = errors.FieldError # type: ignore
metadata_profile = config.SCHEMA_PROFILE["properties"]["fields"]["items"]
metadata_duplicate = True
def __setitem__(self, keys, val):
# Sets an item value. keys iterable of nested keys
setitem(functools.reduce(getitem, keys[:-1], self._model), keys[-1],
val)
def state(self):
state = OrderedDict()
for attr in self._state_attrs:
setitem(state, attr, getattr(self, attr))
return state
print("\nConstructive:")
print(" concat(a, b):", operator.concat(a, b))
print("\nSearching:")
print(" contains(a, 1) :", operator.contains(a, 1))
print(' contains(b, "d"):', operator.contains(b, "d"))
print(" countOf(a, 1) :", operator.countOf(a, 1))
print(' countOf(b, "d") :', operator.countOf(b, "d"))
print(" indexOf(a, 5) :", operator.indexOf(a, 1))
print("\nAccess Items:")
print(" getitem(b, 1) :", operator.getitem(b, 1))
print(" getitem(b, slice(1, 3)) :", operator.getitem(b, slice(1, 3)))
print(' setitem(b, 1, "d") :', end=" ")
operator.setitem(b, 1, "d")
print(b)
print(" setitem(a, slice(1, 3), [4, 5]):", end=" ")
operator.setitem(a, slice(1, 3), [4, 5])
print(a)
print("\nDestructive:")
print(" delitem(b, 1) :", end=" ")
operator.delitem(b, 1)
print(b)
print(" delitem(a, slice(1, 3)):", end=" ")
operator.delitem(a, slice(1, 3))
print(a)
# ------------------------------------------------------------------------------
class TestWebElementWrapper:
"""Generic tests for WebElementWrapper.
Note: For some methods, there's a dedicated test class with more involved
tests.
"""
@pytest.fixture
def elem(self):
return get_webelem()
def test_nullelem(self):
"""Test __init__ with a null element."""
with pytest.raises(webelem.IsNullError):
get_webelem(null=True)
def test_double_wrap(self, elem):
"""Test wrapping a WebElementWrapper."""
with pytest.raises(TypeError) as excinfo:
webelem.WebElementWrapper(elem)
assert str(excinfo.value) == "Trying to wrap a wrapper!"
@pytest.mark.parametrize('code', [
str,
lambda e: e[None],
lambda e: operator.setitem(e, None, None),
lambda e: operator.delitem(e, None),
lambda e: None in e,
len,
lambda e: e.is_visible(None),
lambda e: e.rect_on_view(),
lambda e: e.is_writable(),
lambda e: e.is_content_editable(),
lambda e: e.is_editable(),
lambda e: e.is_text_input(),
lambda e: e.debug_text(),
list, # __iter__
])
def test_vanished(self, elem, code):
"""Make sure methods check if the element is vanished."""
elem._elem.isNull.return_value = True
with pytest.raises(webelem.IsNullError):
code(elem)
def test_str(self, elem):
assert str(elem) == 'text'
@pytest.mark.parametrize('is_null, expected', [
(False, "<qutebrowser.browser.webelem.WebElementWrapper "
"html='<fakeelem/>'>"),
(True, '<qutebrowser.browser.webelem.WebElementWrapper html=None>'),
])
def test_repr(self, elem, is_null, expected):
elem._elem.isNull.return_value = is_null
assert repr(elem) == expected
def test_getitem(self):
elem = get_webelem(attributes={'foo': 'bar'})
assert elem['foo'] == 'bar'
def test_getitem_keyerror(self, elem):
with pytest.raises(KeyError):
elem['foo'] # pylint: disable=pointless-statement
def test_setitem(self, elem):
elem['foo'] = 'bar'
assert elem._elem.attribute('foo') == 'bar'
def test_delitem(self):
elem = get_webelem(attributes={'foo': 'bar'})
del elem['foo']
assert not elem._elem.hasAttribute('foo')
def test_setitem_keyerror(self, elem):
with pytest.raises(KeyError):
del elem['foo']
def test_contains(self):
elem = get_webelem(attributes={'foo': 'bar'})
assert 'foo' in elem
assert 'bar' not in elem
@pytest.mark.parametrize('attributes, expected', [
({'one': '1', 'two': '2'}, {'one', 'two'}),
({}, set()),
])
def test_iter(self, attributes, expected):
elem = get_webelem(attributes=attributes)
assert set(elem) == expected
@pytest.mark.parametrize('attributes, length', [
({'one': '1', 'two': '2'}, 2),
({}, 0),
])
def test_len(self, attributes, length):
elem = get_webelem(attributes=attributes)
assert len(elem) == length
@pytest.mark.parametrize('attributes, writable', [
([], True),
(['disabled'], False),
(['readonly'], False),
(['disabled', 'readonly'], False),
])
def test_is_writable(self, attributes, writable):
elem = get_webelem(attributes=attributes)
assert elem.is_writable() == writable
@pytest.mark.parametrize('attributes, expected', [
({}, False),
({'contenteditable': 'false'}, False),
({'contenteditable': 'inherit'}, False),
({'contenteditable': 'true'}, True),
])
def test_is_content_editable(self, attributes, expected):
elem = get_webelem(attributes=attributes)
assert elem.is_content_editable() == expected
@pytest.mark.parametrize('tagname, attributes, expected', [
('input', {}, True),
('textarea', {}, True),
('select', {}, False),
('foo', {'role': 'combobox'}, True),
('foo', {'role': 'textbox'}, True),
('foo', {'role': 'bar'}, False),
('input', {'role': 'bar'}, True),
])
def test_is_text_input(self, tagname, attributes, expected):
elem = get_webelem(tagname=tagname, attributes=attributes)
assert elem.is_text_input() == expected
@pytest.mark.parametrize('xml, expected', [
('<fakeelem/>', '<fakeelem/>'),
('<foo>\n<bar/>\n</foo>', '<foo><bar/></foo>'),
('<foo>{}</foo>'.format('x' * 500), '<foo>{}…'.format('x' * 494)),
], ids=['fakeelem', 'newlines', 'long'])
def test_debug_text(self, elem, xml, expected):
elem._elem.toOuterXml.return_value = xml
assert elem.debug_text() == expected
def get_webelem(geometry=None,
frame=None,
*,
null=False,
style=None,
attributes=None,
tagname=None,
classes=None,
parent=None,
js_rect_return=None,
zoom_text_only=False):
"""Factory for WebKitElement objects based on a mock.
Args:
geometry: The geometry of the QWebElement as QRect.
frame: The QWebFrame the element is in.
null: Whether the element is null or not.
style: A dict with the styleAttributes of the element.
attributes: Boolean HTML attributes to be added.
tagname: The tag name.
classes: HTML classes to be added.
js_rect_return: If None, what evaluateJavaScript returns is based on
geometry. If set, the return value of
evaluateJavaScript.
zoom_text_only: Whether zoom.text_only is set in the config
"""
elem = mock.Mock()
elem.isNull.return_value = null
elem.geometry.return_value = geometry
elem.webFrame.return_value = frame
elem.tagName.return_value = tagname
elem.toOuterXml.return_value = '<fakeelem/>'
elem.toPlainText.return_value = 'text'
elem.parent.return_value = parent
if geometry is not None:
if frame is None:
scroll_x = 0
scroll_y = 0
else:
scroll_x = frame.scrollPosition().x()
scroll_y = frame.scrollPosition().y()
if js_rect_return is None:
if frame is None or zoom_text_only:
zoom = 1.0
else:
zoom = frame.zoomFactor()
elem.evaluateJavaScript.return_value = {
"length": 1,
"0": {
"left": (geometry.left() - scroll_x) / zoom,
"top": (geometry.top() - scroll_y) / zoom,
"right": (geometry.right() - scroll_x) / zoom,
"bottom": (geometry.bottom() - scroll_y) / zoom,
"width": geometry.width() / zoom,
"height": geometry.height() / zoom,
}
}
else:
elem.evaluateJavaScript.return_value = js_rect_return
attribute_dict = {}
if attributes is None:
pass
elif not isinstance(attributes, collections.abc.Mapping):
attribute_dict.update({e: None for e in attributes})
else:
attribute_dict.update(attributes)
elem.hasAttribute.side_effect = lambda k: k in attribute_dict
elem.attribute.side_effect = lambda k: attribute_dict.get(k, '')
elem.setAttribute.side_effect = (
lambda k, v: operator.setitem(attribute_dict, k, v))
elem.removeAttribute.side_effect = attribute_dict.pop
elem.attributeNames.return_value = list(attribute_dict)
if classes is not None:
elem.classes.return_value = classes.split(' ')
else:
elem.classes.return_value = []
style_dict = {
'visibility': '',
'display': '',
'foo': 'bar',
'opacity': '100'
}
if style is not None:
style_dict.update(style)
def _style_property(name, strategy):
"""Helper function to act as styleProperty method."""
if strategy != QWebElement.ComputedStyle:
raise ValueError("styleProperty called with strategy != "
"ComputedStyle ({})!".format(strategy))
return style_dict[name]
elem.styleProperty.side_effect = _style_property
tab = mock.Mock(autospec=browsertab.AbstractTab)
tab.is_deleted.return_value = False
wrapped = webkitelem.WebKitElement(elem, tab=tab)
return wrapped
def __setitem__(self, key, val):
if key not in self._dict.keys():
raise KeyError(key)
oldval = self[key]
setitem(self._dict, key, val)
self.model_notification.emit(self._keys + tuple([key]), oldval, val)
def preprocess(ogString):
newString = ogString
#count the starting number of rows and tokens per row and returns a dataframe with this info
startText = ogString
startingCount = pd.DataFrame(columns=['row', 'startingTokenCount'])
for g, h in enumerate(startText.split("\n")):
d = {'row': [(g + 1)], 'startingTokenCount': [len(h.split())]}
frame = pd.DataFrame(data=d)
startingCount = startingCount.append(frame)
startingCount = startingCount.reset_index(drop=True)
numberOfStartingRows = len(startingCount)
newString = newString.lower()
newString = re.sub(r"\n", ' \n ', newString)
#### listing of symbols that seperate dates including spaces
dateSeparators = ['\-\s|\/\s|\.\s|\,\s|\s']
#### listing of symbols that seperate dates without spaces
dateSeparatorsNoSpace = ['\-|\/|\.|\,']
#### listing of ways to represent days of the week
daysOfWeek = [
'monday|tuesday|wednesday|thursday|friday|saturday|sunday|mon|mo|tue|tues|tu|wed|we|thur|thurs|th|thu|fri|fr|sat|sa|sun|su|\d\d|\d'
]
#### listing of ways to represent months
monthList = [
'january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|may|jun|jul|aug|sept|oct|nov|dec|\d\d|\d'
]
#### listing of ways to represent a year
yearFormat = ['\d\d\d\d|\d\d']
#### listing of ordinals for days of the week
dayOfWeekOrdinalIndicator = ['th|st|nd|rd|']
#### listing of our date lists
dateFormat = [(yearFormat[0]), (daysOfWeek[0]), (monthList[0])]
#### set counter to start at 4 as the max tokens for a date format example: wednesday, nov 21st 1982
counter = 4
#### use a while loop that stops at 2 tokens for a date format to iterate
while counter > 1:
#### use a cartesian product to loop through all combinations of our date formats
combo = itertools.product(dateFormat, repeat=counter)
#### for loop to process the cartesian product loop
for tt in combo:
#### need to make 2 copies of the cartesian product list one for with spaces and one w/o
makeList = list(tt)
makeList1 = list(tt)
#### loop to convert dates formats with spaces to DATE matching the same amount of date format tokens
howLong = len(makeList1)
step = 1
while step <= howLong + 1:
makeList1.insert(step, dateSeparators[0])
step += (1 + 1)
index4DaysOfWeek = [
i for i, x in enumerate(makeList1) if x == daysOfWeek[0]
]
addOrdinal = [
"(" + daysOfWeek[0] + ")(" + dayOfWeekOrdinalIndicator[0] + ")"
]
for location, replacement in zip(index4DaysOfWeek,
(addOrdinal[0:] *
(len(index4DaysOfWeek)))):
setitem(makeList1, location, replacement)
if len(makeList1) == 7:
newString = re.sub(
r"((" + makeList1[0] + ")(" + makeList1[1] + ")(" +
makeList1[2] + ")(" + makeList1[3] + ")(" + makeList1[4] +
")(" + makeList1[5] + ")(" + makeList1[6] + "))",
'DATE DATE DATE DATE', newString)
#newString = re.sub('\b\d+\b', 'NUM',newString)
else:
if len(makeList1) == 5:
newString = re.sub(
r"((" + makeList1[0] + ")(" + makeList1[1] + ")(" +
makeList1[2] + ")(" + makeList1[3] + ")(" +
makeList1[4] + "))", 'DATE DATE DATE', newString)
else:
if len(makeList1) == 4:
newString = re.sub(
r"((" + makeList1[0] + ")(" + makeList1[1] + ")(" +
makeList1[2] + "))", 'DATE DATE', newString)
else:
pass
#### loop to convert dates formats with no spaces to DATE
howLong1 = len(makeList)
step1 = 1
while step1 <= howLong1 + 1:
makeList.insert(step1, dateSeparatorsNoSpace[0])
step1 += (1 + 1)
if len(makeList) == 7:
newString = re.sub(
r"((" + makeList[0] + ")(" + makeList[1] + ")(" +
makeList[2] + ")(" + makeList[3] + ")(" + makeList[4] +
")(" + makeList[5] + ")(" + makeList[6] + "))", 'DATE',
newString)
else:
if len(makeList) == 5:
newString = re.sub(
r"((" + makeList[0] + ")(" + makeList[1] + ")(" +
makeList[2] + ")(" + makeList[3] + ")(" + makeList[4] +
"))", 'DATE', newString)
else:
if len(makeList) == 4:
newString = re.sub(
r"((" + makeList[0] + ")(" + makeList[1] + ")(" +
makeList[2] + "))", 'DATE', newString)
else:
pass
counter = counter - 1
#clean up DATES that have characters on either end
newString = re.sub('\S*\w*DATE\w*\S*', 'DATE', newString)
#newString = re.sub('\b\d+\b', 'NUM',newString)
#### this ends the date processing section
#####################################################################################
#### this starts the number processing section
#################################################################################################
#### the following finds our target list idenifers and converts to unicode characters
numberMap1 = {
r'\d+\.\)': 'NUM',
'1.)': u'\u0241',
'2.)': u'\u0242',
'3.)': u'\u0243',
'4.)': u'\u0244',
'5.)': u'\u0245',
'6.)': u'\u0246',
'7.)': u'\u0247',
'8.)': u'\u0248',
'9.)': u'\u0249',
'10.)': u'\u0250',
'11.)': u'\u0251',
'12.)': u'\u0252',
'13.)': u'\u0253',
'14.)': u'\u0254',
'15.)': u'\u0255',
'16.)': u'\u0256',
'17.)': u'\u0257',
'18.)': u'\u0258',
'19.)': u'\u0259',
'20.)': u'\u0260'
}
replaceNumberMap1 = re.sub(r'\d+\.\)',
lambda x: numberMap1.get(x.group(), x.group(0)),
newString)
numberMap2 = {
r'\d+\)': 'NUM',
'1)': u'\u0261',
'2)': u'\u0262',
'3)': u'\u0263',
'4)': u'\u0264',
'5)': u'\u0265',
'6)': u'\u0266',
'7)': u'\u0267',
'8)': u'\u0268',
'9)': u'\u0269',
'10)': u'\u0270',
'11)': u'\u0271',
'12)': u'\u0272',
'13)': u'\u0273',
'14)': u'\u0274',
'15)': u'\u0275',
'16)': u'\u0276',
'17)': u'\u0277',
'18)': u'\u0278',
'19)': u'\u0279',
'20)': u'\u0280'
}
replaceNumberMap2 = re.sub(r'\d+\)',
lambda x: numberMap2.get(x.group(), x.group(0)),
replaceNumberMap1)
numberMap4 = {
r'd+\.\s': 'NUM',
'1. ': u'\u0200',
'2. ': u'\u0201',
'3. ': u'\u0202',
'4. ': u'\u0203',
'5. ': u'\u0204',
'6. ': u'\u0205',
'7. ': u'\u0206',
'8. ': u'\u0207',
'9. ': u'\u0208',
'10. ': u'\u0209',
'11. ': u'\u0210',
'12. ': u'\u0211',
'13. ': u'\u0212',
'14. ': u'\u0213',
'15. ': u'\u0214',
'16. ': u'\u0215',
'17. ': u'\u0216',
'18. ': u'\u0217',
'19. ': u'\u0218',
'20. ': u'\u0219'
}
replaceNumberMap4 = re.sub(r'\d+\.\s',
lambda x: numberMap4.get(x.group(), x.group(0)),
replaceNumberMap2)
#### convert all remining numbers to NUM
numbers2NUM1 = re.sub(r'(\d+)', 'NUM', replaceNumberMap4)
### remap the mapping of list identifers back to their original form
remapNumberMap1 = {
u'\u0241': '1.)',
u'\u0242': '2.)',
u'\u0243': '3.)',
u'\u0244': '4.)',
u'\u0245': '5.)',
u'\u0246': '6.)',
u'\u0247': '7.)',
u'\u0248': '8.)',
u'\u0249': '9.)',
u'\u0250': '10.)',
u'\u0251': '11.)',
u'\u0252': '12.)',
u'\u0253': '13.)',
u'\u0254': '14.)',
u'\u0255': '15.)',
u'\u0256': '16.)',
u'\u0257': '17.)',
u'\u0258': '18.)',
u'\u0259': '19.)',
u'\u0260': '20.)'
}
remapNumberMap11 = u'\u0241'
remapNumberMap12 = u'\u0242'
remapNumberMap13 = u'\u0243'
remapNumberMap14 = u'\u0244'
remapNumberMap15 = u'\u0245'
remapNumberMap16 = u'\u0246'
remapNumberMap17 = u'\u0247'
remapNumberMap18 = u'\u0248'
remapNumberMap19 = u'\u0249'
remapNumberMap110 = u'\u0250'
remapNumberMap111 = u'\u0251'
remapNumberMap112 = u'\u0252'
remapNumberMap113 = u'\u0253'
remapNumberMap114 = u'\u0254'
remapNumberMap115 = u'\u0255'
remapNumberMap116 = u'\u0256'
remapNumberMap117 = u'\u0257'
remapNumberMap118 = u'\u0258'
remapNumberMap119 = u'\u0259'
remapNumberMap120 = u'\u0260'
replaceRemapNumberMap1 = re.sub(remapNumberMap11, '1.)', numbers2NUM1)
replaceRemapNumberMap11 = re.sub(remapNumberMap12, '2.)',
replaceRemapNumberMap1)
replaceRemapNumberMap12 = re.sub(remapNumberMap13, '3.)',
replaceRemapNumberMap11)
replaceRemapNumberMap13 = re.sub(remapNumberMap14, '4.)',
replaceRemapNumberMap12)
replaceRemapNumberMap14 = re.sub(remapNumberMap15, '5.)',
replaceRemapNumberMap13)
replaceRemapNumberMap15 = re.sub(remapNumberMap16, '6.)',
replaceRemapNumberMap14)
replaceRemapNumberMap16 = re.sub(remapNumberMap17, '7.)',
replaceRemapNumberMap15)
replaceRemapNumberMap17 = re.sub(remapNumberMap18, '8.)',
replaceRemapNumberMap16)
replaceRemapNumberMap18 = re.sub(remapNumberMap19, '9.)',
replaceRemapNumberMap17)
replaceRemapNumberMap19 = re.sub(remapNumberMap110, '10.)',
replaceRemapNumberMap18)
replaceRemapNumberMap110 = re.sub(remapNumberMap111, '11.)',
replaceRemapNumberMap19)
replaceRemapNumberMap111 = re.sub(remapNumberMap112, '12.)',
replaceRemapNumberMap110)
replaceRemapNumberMap112 = re.sub(remapNumberMap113, '13.)',
replaceRemapNumberMap111)
replaceRemapNumberMap113 = re.sub(remapNumberMap114, '14.)',
replaceRemapNumberMap112)
replaceRemapNumberMap114 = re.sub(remapNumberMap115, '15.)',
replaceRemapNumberMap113)
replaceRemapNumberMap115 = re.sub(remapNumberMap116, '16.)',
replaceRemapNumberMap114)
replaceRemapNumberMap116 = re.sub(remapNumberMap117, '17.)',
replaceRemapNumberMap115)
replaceRemapNumberMap117 = re.sub(remapNumberMap118, '18.)',
replaceRemapNumberMap116)
replaceRemapNumberMap118 = re.sub(remapNumberMap119, '19.)',
replaceRemapNumberMap117)
replaceRemapNumberMap119 = re.sub(remapNumberMap120, '20.)',
replaceRemapNumberMap118)
remapNumberMap2 = {
u'\u0261': '1)',
u'\u0262': '2)',
u'\u0263': '3)',
u'\u0264': '4)',
u'\u0265': '5)',
u'\u0266': '6)',
u'\u0267': '7)',
u'\u0268': '8)',
u'\u0269': '9)',
u'\u0270': '10)',
u'\u0271': '11)',
u'\u0272': '12)',
u'\u0273': '13)',
u'\u0274': '14)',
u'\u0275': '15)',
u'\u0276': '16)',
u'\u0277': '17)',
u'\u0278': '18)',
u'\u0279': '19)',
u'\u0280': '20)'
}
remapNumberMap21 = u'\u0261'
remapNumberMap22 = u'\u0262'
remapNumberMap23 = u'\u0263'
remapNumberMap24 = u'\u0264'
remapNumberMap25 = u'\u0265'
remapNumberMap26 = u'\u0266'
remapNumberMap27 = u'\u0267'
remapNumberMap28 = u'\u0268'
remapNumberMap29 = u'\u0269'
remapNumberMap210 = u'\u0270'
remapNumberMap211 = u'\u0271'
remapNumberMap212 = u'\u0272'
remapNumberMap213 = u'\u0273'
remapNumberMap214 = u'\u0274'
remapNumberMap215 = u'\u0275'
remapNumberMap216 = u'\u0276'
remapNumberMap217 = u'\u0277'
remapNumberMap218 = u'\u0278'
remapNumberMap219 = u'\u0279'
remapNumberMap220 = u'\u0280'
replaceRemapNumberMap2 = re.sub(remapNumberMap21, '1)',
replaceRemapNumberMap119)
replaceRemapNumberMap21 = re.sub(remapNumberMap22, '2)',
replaceRemapNumberMap2)
replaceRemapNumberMap22 = re.sub(remapNumberMap23, '3)',
replaceRemapNumberMap21)
replaceRemapNumberMap23 = re.sub(remapNumberMap24, '4)',
replaceRemapNumberMap22)
replaceRemapNumberMap24 = re.sub(remapNumberMap25, '5)',
replaceRemapNumberMap23)
replaceRemapNumberMap25 = re.sub(remapNumberMap26, '6)',
replaceRemapNumberMap24)
replaceRemapNumberMap26 = re.sub(remapNumberMap27, '7)',
replaceRemapNumberMap25)
replaceRemapNumberMap27 = re.sub(remapNumberMap28, '8)',
replaceRemapNumberMap26)
replaceRemapNumberMap28 = re.sub(remapNumberMap29, '9)',
replaceRemapNumberMap27)
replaceRemapNumberMap29 = re.sub(remapNumberMap210, '10)',
replaceRemapNumberMap28)
replaceRemapNumberMap210 = re.sub(remapNumberMap211, '11)',
replaceRemapNumberMap29)
replaceRemapNumberMap211 = re.sub(remapNumberMap212, '12)',
replaceRemapNumberMap210)
replaceRemapNumberMap212 = re.sub(remapNumberMap213, '13)',
replaceRemapNumberMap211)
replaceRemapNumberMap213 = re.sub(remapNumberMap214, '14)',
replaceRemapNumberMap212)
replaceRemapNumberMap214 = re.sub(remapNumberMap215, '15)',
replaceRemapNumberMap213)
replaceRemapNumberMap215 = re.sub(remapNumberMap216, '16)',
replaceRemapNumberMap214)
replaceRemapNumberMap216 = re.sub(remapNumberMap217, '17)',
replaceRemapNumberMap215)
replaceRemapNumberMap217 = re.sub(remapNumberMap218, '18)',
replaceRemapNumberMap216)
replaceRemapNumberMap218 = re.sub(remapNumberMap219, '19)',
replaceRemapNumberMap217)
replaceRemapNumberMap219 = re.sub(remapNumberMap220, '20)',
replaceRemapNumberMap218)
remapNumberMap4 = {
u'\u0200': '1. ',
u'\u0201': '2. ',
u'\u0202': '3. ',
u'\u0203': '4. ',
u'\u0204': '5. ',
u'\u0205': '6. ',
u'\u0206': '7. ',
u'\u0207': '8. ',
u'\u0208': '9. ',
u'\u0209': '10. ',
u'\u0210': '11. ',
u'\u0211': '12. ',
u'\u0212': '13. ',
u'\u0213': '14. ',
u'\u0214': '15. ',
u'\u0215': '16. ',
u'\u0216': '17. ',
u'\u0217': '18. ',
u'\u0218': '19. ',
u'\u0219': '20. '
}
remapNumberMap41 = u'\u0200'
remapNumberMap42 = u'\u0201'
remapNumberMap43 = u'\u0202'
remapNumberMap44 = u'\u0203'
remapNumberMap45 = u'\u0204'
remapNumberMap46 = u'\u0205'
remapNumberMap47 = u'\u0206'
remapNumberMap48 = u'\u0207'
remapNumberMap49 = u'\u0208'
remapNumberMap410 = u'\u0209'
remapNumberMap411 = u'\u0210'
remapNumberMap412 = u'\u0211'
remapNumberMap413 = u'\u0212'
remapNumberMap414 = u'\u0213'
remapNumberMap415 = u'\u0214'
remapNumberMap416 = u'\u0215'
remapNumberMap417 = u'\u0216'
remapNumberMap418 = u'\u0217'
remapNumberMap419 = u'\u0218'
remapNumberMap420 = u'\u0219'
replaceRemapNumberMap4 = re.sub(remapNumberMap41, '1. ',
replaceRemapNumberMap219)
replaceRemapNumberMap41 = re.sub(remapNumberMap42, '2. ',
replaceRemapNumberMap4)
replaceRemapNumberMap42 = re.sub(remapNumberMap43, '3. ',
replaceRemapNumberMap41)
replaceRemapNumberMap43 = re.sub(remapNumberMap44, '4. ',
replaceRemapNumberMap42)
replaceRemapNumberMap44 = re.sub(remapNumberMap45, '5. ',
replaceRemapNumberMap43)
replaceRemapNumberMap45 = re.sub(remapNumberMap46, '6. ',
replaceRemapNumberMap44)
replaceRemapNumberMap46 = re.sub(remapNumberMap47, '7. ',
replaceRemapNumberMap45)
replaceRemapNumberMap47 = re.sub(remapNumberMap48, '8. ',
replaceRemapNumberMap46)
replaceRemapNumberMap48 = re.sub(remapNumberMap49, '9. ',
replaceRemapNumberMap47)
replaceRemapNumberMap49 = re.sub(remapNumberMap410, '10. ',
replaceRemapNumberMap48)
replaceRemapNumberMap410 = re.sub(remapNumberMap411, '11. ',
replaceRemapNumberMap49)
replaceRemapNumberMap411 = re.sub(remapNumberMap412, '12. ',
replaceRemapNumberMap410)
replaceRemapNumberMap412 = re.sub(remapNumberMap413, '13. ',
replaceRemapNumberMap411)
replaceRemapNumberMap413 = re.sub(remapNumberMap414, '14. ',
replaceRemapNumberMap412)
replaceRemapNumberMap414 = re.sub(remapNumberMap415, '15. ',
replaceRemapNumberMap413)
replaceRemapNumberMap415 = re.sub(remapNumberMap416, '16. ',
replaceRemapNumberMap414)
replaceRemapNumberMap416 = re.sub(remapNumberMap417, '17. ',
replaceRemapNumberMap415)
replaceRemapNumberMap417 = re.sub(remapNumberMap418, '18. ',
replaceRemapNumberMap416)
replaceRemapNumberMap418 = re.sub(remapNumberMap419, '19. ',
replaceRemapNumberMap417)
replaceRemapNumberMap419 = re.sub(remapNumberMap420, '20. ',
replaceRemapNumberMap418)
newString = replaceRemapNumberMap419
newString = re.sub('\S*\w*NUM\w*\S*', 'NUM', newString)
####################################################################################
#count the number of ending rows and tokens per row and returns a dataframe with this info
endText = newString
endCount = pd.DataFrame(columns=['row', 'endingTokenCount'])
for e, m in enumerate(endText.split("\n")):
d = {'row': [(e + 1)], 'endingTokenCount': [len(m.split())]}
endFrame = pd.DataFrame(data=d)
endCount = endCount.append(endFrame)
endCount = endCount.reset_index(drop=True)
numberOfEndRows = len(endCount)
#### merge the two dataframes
merge = pd.merge(startingCount, endCount, how='outer', on='row')
#### create a column that take the difference between the starting and ending token counts per row
merge[
'difference'] = merge['startingTokenCount'] - merge['endingTokenCount']
#### filter the DF so that only rows with a difference are left
mergeDiff = merge.loc[merge['difference'] != 0]
#### if there is an error with the preprocessing report this, if not then report there are no issues
if len(mergeDiff) > 0:
print('Something ain\'t right for this file: \n')
print('the difference in starting len to end is: ',
(numberOfStartingRows - numberOfEndRows))
print(
'\ntable below breaks down the files token input and output by row count for problem rows: \n\n',
mergeDiff)
print('\n\n')
print('problem output text is below \n\n')
for get, this in enumerate(newString.split("\n")):
print(get + 1, this)
else:
print('preprocessed file worked\n')
return (newString)
def metadata_attach(self, name, value):
# Using standalone `setitem` without a wrapper doesn't work for Python3.6
return setitem(self, name, value)
# 二進位OR運算
print("0x0010 | 0x0011 = operator.or_(0x0010, 0x0011) = ", operator.or_(0x0010, 0x0011))
# 次方運算
print("2 ** 16 = operator.pow(2, 16) = ", operator.pow(2, 16))
# 辨識運算
print("1 is 1 = operator.is_(1,1) = ", operator.is_(1,1))
# 辨識運算
print("1 is not 2 = operator.is_not(1,2) = ", operator.is_not(1,2))
# 以索引指派值
obj = [1,2,3]
operator.setitem(obj, 1, 4)
print("obj[1] = 4 = operator.setitem(obj, 1, 4) = ", obj)
# 以索引刪除值
operator.delitem(obj, 1)
print("del obj[1] = operator.delitem(obj, 1) = ", obj)
# 以索引取值
print("obj[1] = operator.getitem(obj,1) = ", operator.getitem(obj, 1))
# 左移運算
print("16 << 1 = operator.lshift(16, 1) = ", operator.lshift(16, 1))
# 乘法運算
print("2 * 8 = operator.mul(2, 8) = ", operator.mul(2, 8))
def solution5(v):
list(
map(lambda x: operator.setitem(x, "square", x["width"] * x["length"]),
v))
return v
def rsetitem(obj, attr, val):
pre, _, post = attr.rpartition(".")
return operator.setitem(rgetitem(obj, pre) if pre else obj, post, val)
def get_agents_without_group(offset=0,
limit=common.database_limit,
sort=None,
search=None,
select=None):
"""
Gets the agents in a group
:param group_id: Group ID.
:param offset: First item to return.
:param limit: Maximum number of items to return.
:param sort: Sorts the items. Format: {"fields":["field1","field2"],"order":"asc|desc"}.
:param search: Looks for items with the specified string.
:return: Dictionary: {'items': array of items, 'totalItems': Number of items (without applying the limit)}
"""
# Connect DB
db_global = glob(common.database_path_global)
if not db_global:
raise WazuhException(1600)
conn = Connection(db_global[0])
valid_select_fiels = {
"id", "name", "ip", "last_keepalive", "os_name", "os_version",
"os_platform", "os_uname", "version", "config_sum", "merged_sum",
"manager_host", "status"
}
# fields like status need to retrieve others to be properly computed.
dependent_select_fields = {'status': {'last_keepalive', 'version'}}
search_fields = {
"id", "name", "os_name", "ip", "status", "version", "os_platform",
"manager_host"
}
# Init query
query = "SELECT {0} FROM agent WHERE `group` IS NULL AND id != 0"
fields = {'id': 'id', 'name': 'name'} # field: db_column
request = {}
# Select
if select:
select_fields_param = set(select['fields'])
if not select_fields_param.issubset(valid_select_fiels):
uncorrect_fields = select_fields_param - valid_select_fiels
raise WazuhException(1724, "Allowed select fields: {0}. Fields {1}".\
format(', '.join(list(valid_select_fiels)), ', '.join(uncorrect_fields)))
select_fields = select_fields_param
else:
select_fields = valid_select_fiels
# add dependent select fields to the database select query
db_select_fields = set()
for dependent, dependent_fields in dependent_select_fields.items():
if dependent in select_fields:
db_select_fields |= dependent_fields
db_select_fields |= (select_fields - set(dependent_select_fields.keys()))
# Search
if search:
query += " AND NOT" if bool(search['negation']) else ' AND'
query += " (" + " OR ".join(x + ' LIKE :search'
for x in search_fields) + " )"
request['search'] = '%{0}%'.format(
int(search['value']) if search['value'].isdigit(
) else search['value'])
# Count
conn.execute(query.format('COUNT(*)'), request)
data = {'totalItems': conn.fetch()[0]}
# Sorting
if sort:
if sort['fields']:
allowed_sort_fields = db_select_fields
# Check if every element in sort['fields'] is in allowed_sort_fields.
if not set(sort['fields']).issubset(allowed_sort_fields):
raise WazuhException(1403, 'Allowed sort fields: {0}. Fields: {1}'.\
format(allowed_sort_fields, sort['fields']))
order_str_fields = [
'{0} {1}'.format(fields[i], sort['order'])
for i in sort['fields']
]
query += ' ORDER BY ' + ','.join(order_str_fields)
else:
query += ' ORDER BY id {0}'.format(sort['order'])
else:
query += ' ORDER BY id ASC'
# OFFSET - LIMIT
if limit:
query += ' LIMIT :offset,:limit'
request['offset'] = offset
request['limit'] = limit
# Data query
conn.execute(query.format(','.join(db_select_fields)), request)
non_nested = [{field:tuple_elem for field,tuple_elem \
in zip(db_select_fields, tuple) if tuple_elem} for tuple in conn]
if 'id' in select_fields:
map(lambda x: setitem(x, 'id', str(x['id']).zfill(3)), non_nested)
if 'status' in select_fields:
try:
map(
lambda x: setitem(
x, 'status',
Agent.calculate_status(x['last_keepalive'], x['version'] ==
None)), non_nested)
except KeyError:
pass
# return only the fields requested by the user (saved in select_fields) and not the dependent ones
non_nested = [{k: v
for k, v in d.items() if k in select_fields}
for d in non_nested]
data['items'] = [plain_dict_to_nested_dict(d, ['os']) for d in non_nested]
return data
def exec(self):
numbers = list(range(1, 11))
# ----------------------------------------------
# operator.add
# -------------------------------
# x + y と同じ
# ----------------------------------------------
pr('operator.add', list(it.accumulate(numbers, func=ope.add)))
# ----------------------------------------------
# operator.add
# -------------------------------
# x - y と同じ
# ----------------------------------------------
pr('operator.sub', list(it.accumulate(numbers, func=ope.sub)))
# ----------------------------------------------
# operator.mul
# -------------------------------
# x * y と同じ
# ----------------------------------------------
pr('operator.mul', list(it.accumulate(numbers, func=ope.mul)))
# ----------------------------------------------
# operator.floordiv
# -------------------------------
# x // y と同じ
# ----------------------------------------------
pr('operator.floordiv', list(it.accumulate(numbers, func=ope.floordiv)))
# ----------------------------------------------
# operator.truediv
# -------------------------------
# x / y と同じ
# ----------------------------------------------
pr('operator.truediv', list(it.accumulate(numbers, func=ope.truediv)))
# ----------------------------------------------
# operator.mod
# -------------------------------
# x % y と同じ
# ----------------------------------------------
pr('operator.mod', list(it.accumulate(numbers, func=ope.mod)))
# ----------------------------------------------
# operator.attrgetter
# -------------------------------
# x.y と同じ
# 一度に複数のメンバーを指定することも可能
# ネストしたメンバー指定も可能
# ----------------------------------------------
Rec = coll.namedtuple('Rec', 'id name subrec')
rec01 = Rec(1, 'name01', Rec(10, 'name10', None))
ope_getter = ope.attrgetter('name')
pr('operator.attrgetter', ope_getter(rec01))
pr('operator.attrgetter', ope.attrgetter('name', 'subrec')(rec01))
pr('operator.attrgetter', ope.attrgetter('subrec.name')(rec01))
# ----------------------------------------------
# operator.getitem
# -------------------------------
# x[y] と同じ
# ----------------------------------------------
dict01 = dict(id=100, name='name100')
pr('operator.getitem', ope.getitem(dict01, 'name'))
# tuple はインデックスアクセスしか出来ないのでインデックスを指定する
pr('operator.getitem', ope.getitem(rec01, 1))
# ----------------------------------------------
# operator.setitem
# -------------------------------
# x[y] = z と同じ
# ----------------------------------------------
ope.setitem(dict01, 'name', 'hello world')
pr('operator.setitem', dict01)
try:
ope.setitem(rec01, 1, 'hello world')
pr('operator.setitem', rec01)
except TypeError as e:
pr('namedtupleはイミュータブルなのでsetitemは無理', e)
# ----------------------------------------------
# operator.methodcaller
# -------------------------------
# x.y() と同じ
# ----------------------------------------------
WithFuncRec = coll.namedtuple('WithFuncRec', 'id name func')
with_func = WithFuncRec(1, 'with func', func=ope.add)
ope_caller = ope.methodcaller('func', 10, 20)
pr('operator.methodcaller', ope_caller(with_func))
def solution5(rooms):
from operator import setitem
rooms = list(
map(lambda x: setitem(x, 'square', x['width'] * x['length']), rooms))
return rooms
