def processBiGrams(stemList, wordList=None, biGramSet=beerNGrams): """ Transforms all BiGrams in word list from pairs of tokens into single tokens. :param stemList: the list of tokens :type stemList: list of unicode :param wordList: the list of words :type wordList: list of unicode :param biGramSet: the set of biGrams to evaluate against. They are strings, not tuples :type biGramSet: set of unicode :rtype tuple """ newSList = [] newWList = [] consumed = False if wordList is None: wordList = stemList if isgenerator(stemList): stemList = list(stemList) if isgenerator(wordList): wordList = list(wordList) if len(stemList): for s1, s2, w1, w2 in izip(stemList[:-1], stemList[1:], wordList[:-1], wordList[1:]): if consumed: consumed = False continue if u'{} {}'.format(s1, s2) in biGramSet: newSList.append(u'{} {}'.format(s1, s2)) newWList.append(u'{} {}'.format(w1, w2)) consumed = True else: newSList.append(s1) newWList.append(w1) if not consumed: newSList.append(stemList[-1]) newWList.append(wordList[-1]) return newSList, newWList
def gen_members_parent(self, ctx, cls, inst, parent, tag_name, subelts): if isinstance(parent, etree._Element): elt = etree.SubElement(parent, tag_name) elt.extend(subelts) ret = self._get_members_etree(ctx, cls, inst, elt) if isgenerator(ret): try: while True: y = (yield) # may throw Break ret.send(y) except Break: try: ret.throw(Break()) except StopIteration: pass else: with parent.element(tag_name): for e in subelts: parent.write(e) ret = self._get_members_etree(ctx, cls, inst, parent) if isgenerator(ret): try: while True: y = (yield) ret.send(y) except Break: try: ret.throw(Break()) except StopIteration: pass
def test_get_instances_to_sync(self): fake_context = 'fake_context' call_info = {'get_all': 0, 'shuffle': 0} def random_shuffle(_list): call_info['shuffle'] += 1 @staticmethod def instance_get_all_by_filters(context, filters, sort_key, sort_dir, limit, marker): # Pretend we return a full list the first time otherwise we loop # infinitely if marker is not None: return [] self.assertEqual(fake_context, context) self.assertEqual('deleted', sort_key) self.assertEqual('asc', sort_dir) call_info['got_filters'] = filters call_info['get_all'] += 1 instances = [fake_instance.fake_db_instance() for i in range(3)] return instances self.stubs.Set(compute.InstanceList, 'get_by_filters', instance_get_all_by_filters) self.stubs.Set(random, 'shuffle', random_shuffle) instances = cells_utils.get_instances_to_sync(fake_context) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(1, call_info['get_all']) self.assertEqual({}, call_info['got_filters']) self.assertEqual(0, call_info['shuffle']) instances = cells_utils.get_instances_to_sync(fake_context, shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(2, call_info['get_all']) self.assertEqual({}, call_info['got_filters']) self.assertEqual(1, call_info['shuffle']) instances = cells_utils.get_instances_to_sync(fake_context, updated_since='fake-updated-since') self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(3, call_info['get_all']) self.assertEqual({'changes-since': 'fake-updated-since'}, call_info['got_filters']) self.assertEqual(1, call_info['shuffle']) instances = cells_utils.get_instances_to_sync(fake_context, project_id='fake-project', updated_since='fake-updated-since', shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(3, len([x for x in instances])) self.assertEqual(4, call_info['get_all']) self.assertEqual({'changes-since': 'fake-updated-since', 'project_id': 'fake-project'}, call_info['got_filters']) self.assertEqual(2, call_info['shuffle'])
def array_to_parent(self, ctx, cls, inst, parent, name, **kwargs): name = cls.get_type_name() if isinstance(inst, PushBase): while True: sv = (yield) print(sv) ret = self.to_parent(ctx, cls, sv, parent, name, from_arr=True, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as e: try: ret.throw(e) except StopIteration: pass else: for sv in inst: ret = self.to_parent(ctx, cls, sv, parent, name, from_arr=True, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as e: try: ret.throw(e) except StopIteration: pass
def complex_model_to_parent(self, ctx, cls, inst, parent, name, tr_child=False, **kwargs): attrs = {} if tr_child is False: with parent.element('tr', attrs): ret = self._get_members(ctx, cls, inst, parent, tr_child=True, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as b: try: ret.throw(b) except StopIteration: pass else: if self.table_name_attr is not None: attrs[self.table_name_attr] = name with parent.element('td', attrs): ret = self.subserialize(ctx, cls, inst, parent, None, name) if isgenerator(ret): while True: sv2 = (yield) ret.send(sv2)
def _gen_table(self, ctx, cls, inst, parent, name, gen_rows, **kwargs): logger.debug("Generate table for %r", cls) attrib = {} if self.table_name_attr is not None: attrib[self.table_name_attr] = cls.get_type_name() with parent.element('table', attrib, nsmap=NSMAP): if self.produce_header: self._gen_header(ctx, cls, name, parent) with parent.element('tbody'): ret = gen_rows(ctx, cls, inst, parent, name, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as b: try: ret.throw(b) except StopIteration: pass ret = self.extend_table(ctx, cls, parent, name, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as b: try: ret.throw(b) except StopIteration: pass
def wrap_table(self, ctx, cls, inst, parent, name, gen_rows, **kwargs): # If this is direct child of an array, table is already set up in # array_to_parent. if self.label: div_attrib = self._gen_label_wrapper_class(ctx, cls, name) label = self._gen_label_for(ctx, cls, name) with parent.element('div', attrib=div_attrib): parent.write(label) ret = self._gen_table(ctx, cls, inst, parent, name, gen_rows, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as b: try: ret.throw(b) except StopIteration: pass else: ret = self._gen_table(ctx, cls, inst, parent, name, gen_rows, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as b: try: ret.throw(b) except StopIteration: pass
def decorated(*args, **kwargs): auth = UserInfo(get_config_parameter('IRODSUSER'), get_config_parameter('IRODSPASS')) conn = connection_pool.get_connection(auth) if conn is None: raise NotAuthorizedException('Invalid credentials') kwargs.update({'conn': conn.connection}) try: res = f(*args, **kwargs) except: connection_pool.release_connection(conn) raise if isgenerator(res): #current_app.logger.debug('typical ls() case encountered') return wrap_generator(res, connection_pool, conn) elif isinstance(res, tuple): #current_app.logger.debug('typical read() case encountered') if not any(map(isgenerator, res)): connection_pool.release_connection(conn) return res else: # generator is in the result tuple wrapped_res = [wrap_generator(i, connection_pool, conn) if isgenerator(i) else i for i in res] return wrapped_res else: #current_app.logger.debug('other case encountered') connection_pool.release_connection(conn) return res
def test_builtin_render_indirect(self): """ Test indirect template render path with `Templates.render` """ l = self.test_construct() result = l.render(self._spawn_handler(), self._spawn_config(app={ 'paths': { 'templates': { 'source': template._FRAMEWORK_TEMPLATE_SOURCES, 'compiled': 'canteen.templates.compiled'}}}), 'base.html', {'var': 'hi'}) assert inspect.isgenerator(result) chunks = [i for i in result] assert '<html></html>' in chunks[0] snippet = l.render(self._spawn_handler(), self._spawn_config(app={ 'paths': { 'templates': { 'source': template._FRAMEWORK_TEMPLATE_SOURCES, 'compiled': 'canteen.templates.compiled'}}}), 'snippets/test.html', {'var': 'hi'}) assert inspect.isgenerator(snippet) chunks = [i for i in snippet] assert '<b></b>' in chunks[0]
def test_multiplex(self): def double(inputs): return (2*x for x in inputs) def square(inputs): return (x*x for x in inputs) def fail(_): raise Exception("Failing before becoming a generator") def fail_later(inputs): for x in inputs: if x % 2: raise Exception else: yield 100 + x inputs = range(4) processors = [double, square] outputs = multiplex(inputs, processors) self.assertTrue(inspect.isgenerator(outputs)) self.assertEquals(list(outputs), [0, 0, 2, 1, 4, 4, 6, 9]) processors = [double, fail, fail_later] outputs = multiplex(inputs, processors, logger=getLogger(self.id())) self.assertTrue(inspect.isgenerator(outputs)) with LogCapture(self.id()) as log: self.assertEquals(list(outputs), [0, 100, 2, 4, 6]) self.assertEquals(len(log.records), 2)
def array_to_parent(self, ctx, cls, inst, parent, name, **kwargs): if issubclass(cls, Array): cls, = cls._type_info.values() name = cls.get_type_name() if isinstance(inst, PushBase): while True: sv = (yield) ret = self.to_parent(ctx, cls, sv, parent, name, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as e: try: ret.throw(e) except StopIteration: pass else: for sv in inst: ret = self.to_parent(ctx, cls, sv, parent, name, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as e: try: ret.throw(e) except StopIteration: pass
def subserialize(self, ctx, cls, inst, parent, ns=None, name=None): attrs = {} if self.table_name_attr is not None: attrs[self.table_name_attr] = name with parent.element('table', attrs): with parent.element('tbody'): if cls.Attributes.max_occurs > 1: ret = self.array_to_parent(ctx, cls, inst, parent, name) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as b: try: ret.throw(b) except StopIteration: pass else: with parent.element('tr'): ret = self.to_parent(ctx, cls, inst, parent, name) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as b: try: ret.throw(b) except StopIteration: pass
def _test_match_basic(matcher, query): extracts = dict( xpaths={'text': 'text()'}, csss={'favorite': '.class1::text'} ) m = matcher(query, **extracts) mg = m(dict(response="<div></div>")) ok_(inspect.isgenerator(mg)) eq_(list(mg), []) # there should be no hits mg = m(dict(response=sample1.response)) ok_(inspect.isgenerator(mg)) hits = list(mg) eq_(len(hits), 3) for hit, a_html, a_text, class1_text in zip( hits, sample1.a_htmls, sample1.a_texts, sample1.class1_texts): ok_(hit['response']) eq_(hit['match'], a_html) eq_(hit['text'], a_text) eq_(hit.get('favorite', None), class1_text) m = matcher(query, min_count=4, **extracts) mg = m(dict(response=sample1.response)) ok_(inspect.isgenerator(mg)) assert_raises(ValueError, list, mg) m = matcher(query, max_count=2, **extracts) mg = m(dict(response=sample1.response)) ok_(inspect.isgenerator(mg)) assert_raises(ValueError, list, mg)
def array_to_cloth(self, ctx, cls, inst, cloth, parent, name=None, **kwargs): if isinstance(inst, PushBase): while True: sv = (yield) ret = self.to_cloth(ctx, cls, sv, cloth, parent, name=name, from_arr=True, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as e: try: ret.throw(e) except StopIteration: pass else: for sv in inst: ret = self.to_cloth(ctx, cls, sv, cloth, parent, from_arr=True, name=name, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as e: try: ret.throw(e) except StopIteration: pass
def tick(self, max_time=None): ''' Process queued messages. :Parameters: - `max_time`: processing time limit so that the event processing does not take too long. not all messages are guranteed to be processed with this limiter :Return: - true: if all messages ready for processing were completed - false: otherwise (i.e.: processing took more than max_time) ''' # swap queues and clear the active_queue self.active_queue, self.processing_queue = self.processing_queue, self.active_queue self.active_queue.clear() coroutines_to_add = [] startTime = time.time() # first process each existing coroutine once exactly self.coroutines = [x for x in self.coroutines if self.process_coroutine(x)] while len(self.processing_queue): # always pop the message off the queue, if there is no listeners for this message yet # then the message will be dropped off the queue msg = self.processing_queue.popleft() # for receivers that handle all messages let them handle this for r in self.message_receiver_map[WildCardMessageType]: res = r.handle_message(msg) # coroutines will be run if inspect.isgenerator(res): c = self.process_coroutine(res) if c: coroutines_to_add.append(c) # now pass msg to message receivers that subscribed to this message type for r in self.message_receiver_map.get(msg.message_type, []): if not self.designated_to_handle(r, msg): continue res = r.handle_message(msg) if inspect.isgenerator(res): c = self.process_coroutine(res) if c: coroutines_to_add.append(c) else: # finish this message if it was handled or had designated receiver if res or msg.receiverID: break if max_time and time.time() - startTime > max_time: break # queue up all pending coroutines self.coroutines.extend(coroutines_to_add) flushed = len(self.processing_queue) == 0 # push any left over messages to the active queue # bottom-up on the processQueue and push to the front of active_queue if not flushed: while len(self.processing_queue): self.active_queue.appendleft(self.processing_queue.pop()) return flushed
def array_to_parent(self, ctx, cls, inst, parent, name, **kwargs): with parent.element('div'): if issubclass(cls, ComplexModelBase): ret = super(HtmlRowTable, self).array_to_parent( ctx, cls, inst, parent, name, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as b: try: ret.throw(b) except StopIteration: pass else: table_attrib = {} if self.table_name_attr: table_attrib = {self.table_name_attr: name} if self.table_width is not None: table_attrib['width'] = self.table_width with parent.element('table', table_attrib): tr_attrib = {} if self.row_class is not None: tr_attrib['class'] = self.row_class with parent.element('tr', tr_attrib): if self.header: parent.write(E.th(self.trc(cls, ctx.locale, cls.get_type_name()))) td_attrs = {} if self.cell_class is not None: self.add_html_attr('class', td_attrs, self.cell_class) self.add_field_attrs(td_attrs, name, cls) cls_attrs = self.get_cls_attrs(cls) if cls_attrs.hidden: self.add_style(td_attrs, 'display:None') with parent.element('td', td_attrs): with parent.element('table'): ret = super(HtmlRowTable, self) \ .array_to_parent(ctx, cls, inst, parent, name, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as b: try: ret.throw(b) except StopIteration: pass
def test_get_instances_to_sync(self): fake_context = 'fake_context' call_info = {'get_all': 0, 'shuffle': 0} def random_shuffle(_list): call_info['shuffle'] += 1 @staticmethod def instance_get_all_by_filters(context, filters, sort_key, sort_dir): self.assertEqual(context, fake_context) self.assertEqual(sort_key, 'deleted') self.assertEqual(sort_dir, 'asc') call_info['got_filters'] = filters call_info['get_all'] += 1 return ['fake_instance1', 'fake_instance2', 'fake_instance3'] self.stubs.Set(objects.InstanceList, 'get_by_filters', instance_get_all_by_filters) self.stubs.Set(random, 'shuffle', random_shuffle) instances = cells_utils.get_instances_to_sync(fake_context) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 1) self.assertEqual(call_info['got_filters'], {}) self.assertEqual(call_info['shuffle'], 0) instances = cells_utils.get_instances_to_sync(fake_context, shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 2) self.assertEqual(call_info['got_filters'], {}) self.assertEqual(call_info['shuffle'], 1) instances = cells_utils.get_instances_to_sync(fake_context, updated_since='fake-updated-since') self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 3) self.assertEqual(call_info['got_filters'], {'changes-since': 'fake-updated-since'}) self.assertEqual(call_info['shuffle'], 1) instances = cells_utils.get_instances_to_sync(fake_context, project_id='fake-project', updated_since='fake-updated-since', shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertEqual(len([x for x in instances]), 3) self.assertEqual(call_info['get_all'], 4) self.assertEqual(call_info['got_filters'], {'changes-since': 'fake-updated-since', 'project_id': 'fake-project'}) self.assertEqual(call_info['shuffle'], 2)
def _handle_request_coroutine(self): ctx = self.ctx # TODO: Errors are not going through the context and middleware. try: for m in middleware: result = m.start(ctx) if result and inspect.isgenerator(result): # The middleware function is a generator. yield from result if not self.route: raise HttpError(404, ctx.url) ctx.response.body = yield from self.route(self.match, ctx) except HttpError as ex: ctx.response.status = ex.code self.error('HTTP error %s %s url=%s', ex.code, str(ex), ctx.url) self._complete_with_error(ex.code) except: ctx.response.status = 500 self.error('Unhandled error in %s', self.route, exc_info=True) self._complete_with_error(500) for m in reversed(middleware): result = m.complete(ctx) if result and inspect.isgenerator(result): # The middleware function is a generator. yield from result try: ctx.response._send(ctx, self.transport) except: # An error happened while we were sending. (Try to do as little as # possible in _send so there is less chance of getting an exception # there.) At this point our best bet is to abort. logger.error('An error occurred while trying to send: %r', ctx, exc_info=True) # Reset everything self.state = _STATE_READING_HEADERS self.request_length = None self.method = None self.url = None self.headers = None self.route = None self.match = None # Can we put this into the route or request? self.ctx = None if self.buffer: self._process_buffer()
def _handle_call(self, request, fn, m, protocol): if fn != '__batch__': r = self._handle_single_call(request, m) else: # Batch calls r = [] for call in m['calls']: _r = self._handle_single_call(request, call) # If the func invoked above is a streaming function, then fail # this operation as we don't handle streaming functions in batch mode if inspect.isgenerator(_r.get('result')): raise APIException('Cannot invoke streaming API fn in batch mode') if isinstance(_r, dict) and 'success' in _r: _r = _r['result'] if _r['success'] else None r.append(_r) if self.get_status() == 304: return # Get the API function object fnobj = self._get_apifn(fn) if fn != '__batch__' else (lambda: 0) # Set response header based on chosen serialization mechanism mime = getattr(fnobj, 'mime', self.get_mime(protocol)) self.set_header('Content-Type', mime) is_raw = 'raw' in get_fn_tags(fnobj) serializer = (lambda x: x) if is_raw else self.get_serializer(protocol) if fn == '__batch__' or not r['success']: r = serializer(r) self.set_header('Content-Length', len(r)) self.write(r) return result = r['result'] if not inspect.isgenerator(result): # Full response is available - Write it out in one shot r = serializer(r) self.set_header('Content-Length', len(r)) self.write(r) return # Streaming response - iterate and write out for part in result: part = serializer(part) self.write(part) sep = '\n' if is_raw else self.get_record_separator(protocol) if sep: self.write(sep) self.flush()
def test_get_instances_to_sync(self): fake_context = "fake_context" call_info = {"get_all": 0, "shuffle": 0} def random_shuffle(_list): call_info["shuffle"] += 1 def instance_get_all_by_filters(context, filters, sort_key, sort_order): self.assertEqual(context, fake_context) self.assertEqual(sort_key, "deleted") self.assertEqual(sort_order, "asc") call_info["got_filters"] = filters call_info["get_all"] += 1 return ["fake_instance1", "fake_instance2", "fake_instance3"] self.stubs.Set(db, "instance_get_all_by_filters", instance_get_all_by_filters) self.stubs.Set(random, "shuffle", random_shuffle) instances = cells_utils.get_instances_to_sync(fake_context) self.assertTrue(inspect.isgenerator(instances)) self.assertTrue(len([x for x in instances]), 3) self.assertEqual(call_info["get_all"], 1) self.assertEqual(call_info["got_filters"], {}) self.assertEqual(call_info["shuffle"], 0) instances = cells_utils.get_instances_to_sync(fake_context, shuffle=True) self.assertTrue(inspect.isgenerator(instances)) self.assertTrue(len([x for x in instances]), 3) self.assertEqual(call_info["get_all"], 2) self.assertEqual(call_info["got_filters"], {}) self.assertEqual(call_info["shuffle"], 1) instances = cells_utils.get_instances_to_sync(fake_context, updated_since="fake-updated-since") self.assertTrue(inspect.isgenerator(instances)) self.assertTrue(len([x for x in instances]), 3) self.assertEqual(call_info["get_all"], 3) self.assertEqual(call_info["got_filters"], {"changes-since": "fake-updated-since"}) self.assertEqual(call_info["shuffle"], 1) instances = cells_utils.get_instances_to_sync( fake_context, project_id="fake-project", updated_since="fake-updated-since", shuffle=True ) self.assertTrue(inspect.isgenerator(instances)) self.assertTrue(len([x for x in instances]), 3) self.assertEqual(call_info["get_all"], 4) self.assertEqual( call_info["got_filters"], {"changes-since": "fake-updated-since", "project_id": "fake-project"} ) self.assertEqual(call_info["shuffle"], 2)
def print_item(item, level=0, indent=' '): if isfunction(item): lazy = item() print_item(lazy, level+1) elif isgenerator(item): print_recursive(item, level+1) elif isinstance(item, tuple): if isfunction(item[-1]) or isgenerator(item[-1]) or isinstance(item[-1], tuple): print(indent*level+repr(item[:-1])) print_item(item[-1], level+1) else: print(indent*level+repr(item)) else: print(indent*level+repr(item))
def post_process_extensions(self, extensions, resp_obj, request, action_args): for ext in extensions: response = None if inspect.isgenerator(ext): # If it's a generator, run the second half of # processing try: with ResourceExceptionHandler(): response = ext.send(resp_obj) except StopIteration: # Normal exit of generator continue except Fault as ex: response = ex else: # Regular functions get post-processing... try: with ResourceExceptionHandler(): response = ext(req=request, resp_obj=resp_obj, **action_args) except Fault as ex: response = ex # We had a response... if response: return response return None
def test_filter_all_recursive_yields(self): # Test filter_all() allows generators from previous filter_all()s. # filter_all() yields results. We want to make sure that we can # call filter_all() with generators returned from previous calls # to filter_all(). filter_obj_list = ['obj1', 'obj2', 'obj3'] filter_properties = 'fake_filter_properties' base_filter = filters.BaseFilter() self.mox.StubOutWithMock(base_filter, '_filter_one') total_iterations = 200 # The order that _filter_one is going to get called gets # confusing because we will be recursively yielding things.. # We are going to simulate the first call to filter_all() # returning False for 'obj2'. So, 'obj1' will get yielded # 'total_iterations' number of times before the first filter_all() # call gets to processing 'obj2'. We then return 'False' for it. # After that, 'obj3' gets yielded 'total_iterations' number of # times. for x in xrange(total_iterations): base_filter._filter_one('obj1', filter_properties).AndReturn(True) base_filter._filter_one('obj2', filter_properties).AndReturn(False) for x in xrange(total_iterations): base_filter._filter_one('obj3', filter_properties).AndReturn(True) self.mox.ReplayAll() objs = iter(filter_obj_list) for x in xrange(total_iterations): # Pass in generators returned from previous calls. objs = base_filter.filter_all(objs, filter_properties) self.assertTrue(inspect.isgenerator(objs)) self.assertEqual(['obj1', 'obj3'], list(objs))
def test_data_stream_exc(self): fut = asyncio.Future(loop=self.loop) def gen(): yield b'binary data' yield from fut return b' result' req = HttpRequest( 'POST', 'http://python.org/', data=gen(), loop=self.loop) self.assertTrue(req.chunked) self.assertTrue(inspect.isgenerator(req.body)) self.assertEqual(req.headers['transfer-encoding'], 'chunked') @asyncio.coroutine def exc(): yield from asyncio.sleep(0.01, loop=self.loop) fut.set_exception(ValueError) asyncio.async(exc(), loop=self.loop) req.send(self.transport) self.assertRaises( ValueError, self.loop.run_until_complete, req._writer) self.assertRaises(self.transport.close.called)
def _call(): result = getattr(self.client, method)(*args, **kwargs) if inspect.isgenerator(result): # Convert generator results to a list, so that we can # catch any potential exceptions now and retry the call. return list(result) return result
def set_parser(self, parser): """set parser to stream. return parser's DataQueue.""" if self._parser: self.unset_parser() output = DataQueue(loop=self._loop) if self._exception: output.set_exception(self._exception) return output # init parser p = parser(output, self._input) assert inspect.isgenerator(p), 'Generator is required' try: # initialize parser with data and parser buffers next(p) except StopIteration: pass except Exception as exc: output.set_exception(exc) else: # parser still require more data self._parser = p self._output = output if self._eof: self.unset_parser() return output
def test_flatten(): assert isgenerator(flatten([])) assert list(flatten([1])) != [[1]] assert list(flatten([[]])) == [] assert list(flatten([1, [2], [3]])) == [1, 2, 3] assert list(flatten([1, [2, 3], [[4]]])) == [1, 2, 3, 4]
def advising_generator_wrapper_py3(*args, **kwargs): if bind: advisor = advising_function(cutpoint_function, *args, **kwargs) else: advisor = advising_function(*args, **kwargs) if not isgenerator(advisor): raise ExpectedGenerator("advising_function %s did not return a generator." % advising_function) try: advice = next(advisor) while True: logdebug('Got advice %r from %s', advice, advising_function) if advice is Proceed or advice is None or isinstance(advice, Proceed): if isinstance(advice, Proceed): args = advice.args kwargs = advice.kwargs gen = cutpoint_function(*args, **kwargs) try: result = yield from gen except BaseException: advice = advisor.throw(*sys.exc_info()) else: try: advice = advisor.send(result) except StopIteration: return finally: gen.close() elif advice is Return: return elif isinstance(advice, Return): raise StopIteration(advice.value) else: raise UnacceptableAdvice("Unknown advice %s" % advice) finally: advisor.close()
def handle_request(self, message, payload): """Handle a single HTTP request""" now = self._loop.time() if self.readpayload: wsgiinput = io.BytesIO() wsgiinput.write((yield from payload.read())) wsgiinput.seek(0) payload = wsgiinput environ = self.create_wsgi_environ(message, payload) response = self.create_wsgi_response(message) riter = self.wsgi(environ, response.start_response) if isinstance(riter, asyncio.Future) or inspect.isgenerator(riter): riter = yield from riter resp = response.response try: for item in riter: if isinstance(item, asyncio.Future): item = yield from item yield from resp.write(item) yield from resp.write_eof() finally: if hasattr(riter, 'close'): riter.close() if resp.keep_alive(): self.keep_alive(True) self.log_access( message, environ, response.response, self._loop.time() - now)
def _process_data(self, ctxt, version, method, namespace, args): """Process a message in a new thread. If the proxy object we have has a dispatch method (see rpc.dispatcher.RpcDispatcher), pass it the version, method, and args and let it dispatch as appropriate. If not, use the old behavior of magically calling the specified method on the proxy we have here. """ ctxt.update_store() try: rval = self.proxy.dispatch(ctxt, version, method, namespace, **args) # Check if the result was a generator if inspect.isgenerator(rval): for x in rval: ctxt.reply(x, None, connection_pool=self.connection_pool) else: ctxt.reply(rval, None, connection_pool=self.connection_pool) # This final None tells multicall that it is done. ctxt.reply(ending=True, connection_pool=self.connection_pool) except rpc_common.ClientException as e: LOG.debug(_('Expected exception during message handling (%s)') % e._exc_info[1]) ctxt.reply(None, e._exc_info, connection_pool=self.connection_pool, log_failure=False) except Exception: # sys.exc_info() is deleted by LOG.exception(). exc_info = sys.exc_info() LOG.error(_('Exception during message handling'), exc_info=exc_info) ctxt.reply(None, exc_info, connection_pool=self.connection_pool)
def _check_if_gen(val): test = val() if inspect.isgenerator(test): return test else: return val
def _return_executable(val): if inspect.isgenerator(val): return lambda: next(val) return val
def to_cloth(self, ctx, cls, inst, cloth, parent, name=None, from_arr=False, as_attr=False, as_data=False, **kwargs): prot_name = self.__class__.__name__ if issubclass(cls, XmlAttribute): cls = cls.type as_attr = True elif issubclass(cls, XmlData): cls = cls.type as_data = True pushed = False if cloth is None: logger_c.debug("No cloth fround, switching to to_parent...") ret = self.to_parent(ctx, cls, inst, parent, name, **kwargs) else: cls, _ = self.get_polymorphic_target(cls, inst) cls_attrs = self.get_cls_attrs(cls) inst = self._sanitize(cls_attrs, inst) # if instance is None, use the default factory to generate one _df = cls_attrs.default_factory if inst is None and callable(_df): inst = _df() # if instance is still None, use the default value if inst is None: inst = cls_attrs.default # if there's a subprotocol, switch to it subprot = cls_attrs.prot if subprot is not None and not (subprot is self): # we can't do this because subprotocols don't accept cloths. # so we need to enter the cloth, which make it too late to # set attributes. assert not as_attr, "No subprot supported for fields " \ "to be serialized as attributes, use type casting with " \ "customized serializers in the current protocol instead." self._enter_cloth(ctx, cloth, parent, method=cls_attrs.method, skip=as_data) ret = subprot.subserialize(ctx, cls, inst, parent, name, as_attr=as_attr, as_data=as_data, **kwargs) # if there is no subprotocol, try rendering the value else: ret = None # try rendering the null value if inst is None: if cls_attrs.min_occurs > 0: attrs = {} if as_attr: # FIXME: test needed attrs[name] = '' self._enter_cloth(ctx, cloth, parent, attrib=attrs, method=cls_attrs.method) identifier = "%s.%s" % (prot_name, "null_to_cloth") logger_s.debug("Writing '%s' using %s type: %s.", name, identifier, cls.get_type_name()) parent.write(cloth) else: logger_s.debug("Skipping '%s' type: %s because empty.", name, cls.get_type_name()) self._enter_cloth(ctx, cloth, parent, skip=True, method=cls_attrs.method) elif as_data: # we only support XmlData of a primitive.,. is this a # problem? ret = self.to_unicode(cls, inst) if ret is not None: parent.write(ret) elif as_attr: sub_name = cls_attrs.sub_name if sub_name is None: sub_name = name attrs = {sub_name: self.to_unicode(cls, inst)} self._enter_cloth(ctx, cloth, parent, attrib=attrs, method=cls_attrs.method) else: # push the instance at hand to instance stack. this makes it # easier for protocols to make decisions based on parents of # instances at hand. pushed = True logger_c.debug("%s %r pushed %r %r", R("#"), self, cls, inst) ctx.outprot_ctx.inst_stack.append((cls, inst, from_arr)) # try rendering the array value if not from_arr and cls.Attributes.max_occurs > 1: ret = self.array_to_cloth(ctx, cls, inst, cloth, parent, as_attr=as_attr, name=name) else: # try rendering anything else handler = self.rendering_handlers[cls] # disabled for performance reasons # identifier = "%s.%s" % (prot_name, handler.__name__) # from spyne.util.web import log_repr # logger_s.debug("Writing %s using %s for %s. Inst: %r", # name, identifier, cls.get_type_name(), # log_repr(inst, cls, from_array=from_arr)) ret = handler(ctx, cls, inst, cloth, parent, name=name, as_attr=as_attr) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as e: try: ret.throw(e) except (Break, StopIteration, GeneratorExit): pass finally: if pushed: logger_c.debug("%s %r popped %r %r", B("#"), self, cls, inst) ctx.outprot_ctx.inst_stack.pop() else: if pushed: logger_c.debug("%s %r popped %r %r", B("#"), self, cls, inst) ctx.outprot_ctx.inst_stack.pop()
def _step(self, exc=None): assert not self.done(), f'_step(): already done: {self!r}, {exc!r}' if self._must_cancel: if not isinstance(exc, futures.CancelledError): exc = futures.CancelledError() self._must_cancel = False coro = self._coro self._fut_waiter = None _enter_task(self._loop, self) # Call either coro.throw(exc) or coro.send(None). try: if exc is None: # We use the `send` method directly, because coroutines # don't have `__iter__` and `__next__` methods. result = coro.send(None) else: result = coro.throw(exc) except StopIteration as exc: if self._must_cancel: # Task is cancelled right before coro stops. self._must_cancel = False self.set_exception(futures.CancelledError()) else: self.set_result(exc.value) except futures.CancelledError: super().cancel() # I.e., Future.cancel(self). except Exception as exc: self.set_exception(exc) except BaseException as exc: self.set_exception(exc) raise else: blocking = getattr(result, '_asyncio_future_blocking', None) if blocking is not None: # Yielded Future must come from Future.__iter__(). if result._loop is not self._loop: new_exc = RuntimeError( f'Task {self!r} got Future ' f'{result!r} attached to a different loop') self._loop.call_soon(self._step, new_exc) elif blocking: if result is self: new_exc = RuntimeError( f'Task cannot await on itself: {self!r}') self._loop.call_soon(self._step, new_exc) else: result._asyncio_future_blocking = False result.add_done_callback(self._wakeup) self._fut_waiter = result if self._must_cancel: if self._fut_waiter.cancel(): self._must_cancel = False else: new_exc = RuntimeError( f'yield was used instead of yield from ' f'in task {self!r} with {result!r}') self._loop.call_soon(self._step, new_exc) elif result is None: # Bare yield relinquishes control for one event loop iteration. self._loop.call_soon(self._step) elif inspect.isgenerator(result): # Yielding a generator is just wrong. new_exc = RuntimeError( f'yield was used instead of yield from for ' f'generator in task {self!r} with {result}') self._loop.call_soon(self._step, new_exc) else: # Yielding something else is an error. new_exc = RuntimeError(f'Task got bad yield: {result!r}') self._loop.call_soon(self._step, new_exc) finally: _leave_task(self._loop, self) self = None # Needed to break cycles when an exception occurs.
def is_generator(obj): """Return True if ``obj`` is a generator """ return inspect.isgeneratorfunction(obj) or inspect.isgenerator(obj)
sig = inspect.signature(add1) parameters = sig.parameters print( parameters ) #OrderedDict([('args', <Parameter "*args">), ('kwargs', <Parameter "**kwargs">)]) ret = fn(*args, **kwargs) return ret return wrapper @check #add1 = check(add1) def add1(x: int, y: str = 6) -> int: #定义函数add1的返回值的类型注释为int if not (isinstance(x, int) and isinstance(y, str)): return 'you input is wrong' return x + y print(add1(2, 4)) print(add1.__annotations__) #annotation 查看字典的注解 annotation(注释)__annotations__ def fib(n): yield from range(10) print(inspect.isgenerator(fib)) #检查函数是否是生成器 print(inspect.isgeneratorfunction(add1)) #检查函数是否是生成函数 print(inspect.signature(fib)) #检查函数的签名 print(inspect.signature(add1)) #检查函数的签名
def is_iterable(obj): """ :param obj: that we want to determine is a generator :return: True if obj can use next(obj) """ return inspect.isgenerator(obj) or inspect.isgeneratorfunction(obj)
def process_callback(callback_result): "Handle generators in actions" if inspect.isgenerator(callback_result): task = LongRunningTask.alloc().init() task.__dict__['interface'] = callback_result task.performIteration_(None)
def _schedule_proc_now(self, ev): assert isinstance(ev, (Action, Event)) or isgenerator(ev), ev self._current_event_list.append(ev)
def run(self, until: int, extraProcesses=[]) -> None: """ Run simulation for a specified time :note: Can be used to run simulation again after it ends from time when it ends. :note: Simulator restart is performed by new instantiation of the simulator. """ assert until >= self.now, (until, self.now) if until == self.now: return now = self.now time_slot = SimTimeSlot() time_slot.write_only = [] for proc in extraProcesses: assert isgenerator(proc), proc time_slot.write_only.append(proc) # add handle to stop simulation self.schedule(now, time_slot) end_time_slot = SimTimeSlot() end_time_slot.write_only = [raise_StopSimulation(self), ] self.schedule(now + until, end_time_slot) next_time_slot = self._events.pop rtl_sim = self.rtl_simulator _run_event_list = self._run_event_list END = rtl_sim.END_OF_STEP try: # for all events while True: now, time_slot = next_time_slot() self._current_time_slot = time_slot assert now >= self.now, (now, self.now, time_slot, "Can not go back in time") rtl_sim.time = self.now = now # run preinitialization of sim. environment _run_event_list(time_slot.timeslot_begin) time_slot.timeslot_begin = DONE # run resolution of combinational lopps first_run = True while first_run or time_slot.write_only: _run_event_list(time_slot.write_only) time_slot.write_only = None s = rtl_sim.eval() assert s == rtl_sim.COMB_UPDATE_DONE, (self.now, s) if time_slot.comb_read is None: self._current_event_list = time_slot.comb_read = [] else: self._current_event_list = time_slot.comb_read self._eval_rtl_events() _run_event_list(time_slot.comb_read) time_slot.comb_read = None if time_slot.write_only is not None: # we have to reevaluate the combinational logic # if write in this time stamp is required rtl_sim.reset_eval() first_run = False time_slot.write_only = DONE time_slot.comb_read = DONE # run evaluation of rest of the circuit while not rtl_sim.read_only_not_write_only: rtl_sim.eval() if rtl_sim.pending_event_list: if time_slot.comb_read is None: self._current_event_list = time_slot.comb_stable = [] else: self._current_event_list = time_slot.comb_stable self._eval_rtl_events() _run_event_list(time_slot.comb_stable) time_slot.comb_stable = DONE while True: ret = rtl_sim.eval() if rtl_sim.pending_event_list: if time_slot.mem_stable is None: self._current_event_list = time_slot.mem_stable = [] else: self._current_event_list = time_slot.mem_stable self._eval_rtl_events() if ret == END: break _run_event_list(time_slot.mem_stable) time_slot.mem_stable = DONE _run_event_list(time_slot.timeslot_end) time_slot.timeslot_end = DONE rtl_sim.set_write_only() except StopSimumulation: pass finally: rtl_sim.finalize() # to allow tesbenches to peek in to DUT after sim ended rtl_sim.read_only_not_write_only = True
def generator(obj): if not inspect.isgenerator(obj): raise ParameterCheckError( 'Not a generator (return value of function that yields) Got {obj} instead' .format(obj=obj)) return obj
def ff(gen, *args, **kwargs): if not inspect.isgenerator(gen): gen = (gen,) return f(gen, *args, **kwargs)
def __init__(self, gen, func): assert inspect.isgenerator(gen), gen self.gen = gen self.func = func
def evaluate(cls, estimator, dataset, output_dir=None, name=None): """Evaluates an `estimator` on the `mode` benchmark dataset. Args: estimator: `lambda x: mu_x, uncertainty_x`, an uncertainty estimation function, which returns `mean_x` and predictive `uncertainty_x`. dataset: `tf.data.Dataset`, on which dataset to performance evaluation. output_dir: (optional) `str`, directory to save figures. name: (optional) `str`, the name of the method. """ import inspect import tqdm import numpy as np import tensorflow_datasets as tfds import matplotlib.pyplot as plt COLORS = plt.rcParams['axes.prop_cycle'].by_key()['color'] # Containers used for caching performance evaluation y_true = list() y_pred = list() y_uncertainty = list() # Convert to NumPy iterator if necessary ds = dataset if inspect.isgenerator(dataset) else tfds.as_numpy( dataset) for x, y in tqdm.tqdm(ds): # Sample from probabilistic model mean, uncertainty = estimator(x) # Cache predictions y_true.append(y) y_pred.append(mean) y_uncertainty.append(uncertainty) # Use vectorized NumPy containers y_true = np.concatenate(y_true).flatten() y_pred = np.concatenate(y_pred).flatten() y_uncertainty = np.concatenate(y_uncertainty).flatten() fractions = np.asarray([0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) # Metrics for evaluation metrics = zip(["accuracy", "auc"], cls.metrics()) # evaluate evaluation = { metric: cls._evaluate_metric( y_true, y_pred, y_uncertainty, fractions, lambda y_true, y_pred: metric_fn(y_true, y_pred).numpy(), name, ) for (metric, metric_fn) in metrics } # save JSON of evaluation if output_dir is not None: json_evals = dict((key, val.to_dict(orient='record')) for key, val in evaluation.items()) os.makedirs(output_dir, exist_ok=True) json.dump(json_evals, open(os.path.join(output_dir, 'evaluation.json'), 'w')) # print evaluation for metric, evals in evaluation.items(): print(metric) print(evals) return evaluation
def test_start_requests(self): spider = self.spider_class('example.com') start_requests = spider.start_requests() self.assertTrue(inspect.isgenerator(start_requests)) self.assertEqual(list(start_requests), [])
def opt_generator(obj): if obj is not None and not inspect.isgenerator(obj): raise ParameterCheckError( "Not a generator (return value of function that yields) Got {obj} instead" .format(obj=obj)) return obj
def test_check_estimator_generate_only(): all_instance_gen_checks = check_estimator(LogisticRegression(), generate_only=True) assert isgenerator(all_instance_gen_checks)
def Initializer(init, allow_generators=False, treat_sequences_as_mappings=True, arg_not_specified=None): """Standardized processing of Component keyword arguments Component keyword arguments accept a number of possible inputs, from scalars to dictionaries, to functions (rules) and generators. This function standardizes the processing of keyword arguments and returns "initializer classes" that are specialized to the specific data type provided. """ if init.__class__ in native_types: if init is arg_not_specified: return None return ConstantInitializer(init) elif inspect.isfunction(init) or inspect.ismethod(init): if not allow_generators and inspect.isgeneratorfunction(init): raise ValueError("Generator functions are not allowed") # Historically pyomo.core.base.misc.apply_indexed_rule # accepted rules that took only the parent block (even for # indexed components). We will preserve that functionality # here. _args = inspect.getfullargspec(init) _nargs = len(_args.args) if inspect.ismethod(init) and init.__self__ is not None: # Ignore 'self' for bound instance methods and 'cls' for # @classmethods _nargs -= 1 if _nargs == 1 and _args.varargs is None: return ScalarCallInitializer(init) else: return IndexedCallInitializer(init) elif isinstance(init, Mapping): return ItemInitializer(init) elif isinstance(init, Sequence) and not isinstance(init, str): if treat_sequences_as_mappings: return ItemInitializer(init) else: return ConstantInitializer(init) elif inspect.isgenerator(init) or ( (hasattr(init, 'next') or hasattr(init, '__next__')) and not hasattr(init, '__len__')): # This catches generators and iterators (like enumerate()), but # skips "reusable" iterators like range() as well as Pyomo # (finite) Set objects. if not allow_generators: raise ValueError("Generators are not allowed") # Deepcopying generators is problematic (e.g., it generates a # segfault in pypy3 7.3.0). We will immediately expand the # generator into a tuple and then store it as a constant. return ConstantInitializer(tuple(init)) elif isinstance(init, PyomoObject): # TODO: Should IndexedComponent inherit from collections.abc.Mapping? if init.is_component_type() and init.is_indexed(): return ItemInitializer(init) else: return ConstantInitializer(init) elif type(init) is functools.partial: _args = inspect.getfullargspec(init.func) if len(_args.args) - len(init.args) == 1 and _args.varargs is None: return ScalarCallInitializer(init) else: return IndexedCallInitializer(init) elif callable(init) and not isinstance(init, type): # We assume any callable thing could be a functor; but, we must # filter out types, as isfunction() and ismethod() both return # False for type.__call__ return Initializer( init.__call__, allow_generators=allow_generators, treat_sequences_as_mappings=treat_sequences_as_mappings, arg_not_specified=arg_not_specified, ) elif isinstance(init, _ndarray): if init.size == 1: return ConstantInitializer(init[0]) else: return ItemInitializer(init) else: return ConstantInitializer(init)
def isgenerator(o): if isinstance(o, UnboundMethod): o = o._func return inspect.isgeneratorfunction(o) or inspect.isgenerator(o)
def __init__(self, result_proxies): if not isgenerator(result_proxies): result_proxies = iter((result_proxies, )) self.result_proxies = result_proxies self._iter = None
def create_events(pulse_or_list, author=False, server=False, key=False, misp=False, distribution=0, threat_level=4, analysis=2, publish=True, tlp=True, discover_tags=False, to_ids=False, author_tag=False, bulk_tag=None, dedup_titles=False): """ Parse a Pulse or a list of Pulses and add it/them to MISP if server and key are present :param pulse_or_list: a Pulse or list of Pulses as returned by `get_pulses` :param author: Prepend the author to the Pulse name :type author: Boolean :param server: MISP server URL :param key: MISP API key :param misp: MISP connection object :type misp: :class:`pymisp.PyMISP` :param distribution: distribution of the MISP event (0-4) :param threat_level: threat level of the MISP object (1-4) :param analysis: analysis stae of the MISP object (0-2) :param publish: Is the MISP event should be published? :type publish: Boolean :param tlp: Add TLP level tag to event :type tlp: Boolean :param discover_tags: discover MISP tags from Pulse tags :type discover_tags: Boolean :param to_ids: Flag pulse attributes as being sent to an IDS :type to_ids: Boolean :param author_tag: Add the pulse author as an event tag :type author_tag: Boolean :param bulk_tag: A tag that will be added to all events for categorization (e.g. OTX) :type bulk_tag: String :param dedup_titles: Search MISP for an existing event title and update it, rather than create a new one :type dedup_titles: Boolean :return: a dict or a list of dict with the selected attributes """ if not misp and (server and key): log.debug("Connection to MISP instance: {}".format(server)) try: misp = pymisp.PyMISP(server, key, ssl=False, out_type='json', debug=False) except pymisp.PyMISPError as ex: raise ImportException("Cannot connect to MISP instance: {}".format(ex.message)) except Exception as ex: raise ImportException("Cannot connect to MISP instance, unknown exception: {}".format(ex.message)) if discover_tags: def get_tag_name(complete): parts = complete.split('=') if not len(parts): return complete last = parts[-1] if last[0] == '"': last = last[1:] if last[-1] == '"': last = last[:-1] return last.lower() raw_tags = misp.get_all_tags() tags = dict() for tag in raw_tags['Tag']: tags[get_tag_name(tag['name'])] = tag['name'] misp.discovered_tags = tags if isinstance(pulse_or_list, (list, tuple)) or inspect.isgenerator(pulse_or_list): result_event = [] for pulse in pulse_or_list: try: result = create_events(pulse, author=author, server=server, key=key, misp=misp, distribution=distribution, threat_level=threat_level, analysis=analysis, publish=publish, tlp=tlp, to_ids=to_ids, author_tag=author_tag, bulk_tag=bulk_tag, dedup_titles=dedup_titles) result_event.append(result) except: pass return result_event pulse = pulse_or_list if author: event_name = pulse['author_name'] + ' | ' + pulse['name'] else: event_name = pulse['name'] try: dt = date_parser.parse(pulse['modified']) except (ValueError, OverflowError): log.error("Cannot parse Pulse 'modified' date.") dt = datetime.utcnow() event_date = dt.strftime('%Y-%m-%d') log.info("## {name} - {date}".format(name=event_name, date=event_date)) result_event = { 'name': event_name, 'date': event_date, 'tags': list(), 'attributes': { 'hashes': { 'md5': list(), 'sha1': list(), 'sha256': list(), 'imphash': list(), 'pehash': list() }, 'hostnames': list(), 'domains': list(), 'urls': list(), 'ips': list(), 'emails': list(), 'mutexes': list(), 'references': list(), 'cves': list() }, } if misp: if not dedup_titles: event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date, published=publish) else: event='' # Check if username is added to title # Build the title if author: event_name = pulse['author_name'] + ' | ' + pulse['name'] else: event_name = pulse['name'] # Search MISP for the title result = misp.search_index(eventinfo=event_name) if 'message' in result: if result['message'] == "No matches.": event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date, published=publish) else: for evt in result['response']: # If it exists, set 'event' to the event if evt['info'] == event_name: event = {'Event': evt} break if event == '': # Event not found, even though search results were returned # Build new event event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date, published=publish) time.sleep(0.2) if tlp and 'TLP' in pulse: tag = "tlp:{}".format(pulse['TLP']) log.info("\t - Adding tag: {}".format(tag)) tag_event(misp, event, tag) result_event['tags'].append(tag) if author_tag: tag_event(misp, event, pulse['author_name']) if bulk_tag is not None: tag_event(misp, event, bulk_tag) if misp and hasattr(misp, 'discovered_tags') and 'tags' in pulse: for pulse_tag in pulse['tags']: if pulse_tag.lower() in misp.discovered_tags: tag = misp.discovered_tags[pulse_tag.lower()] log.info("\t - Adding tag: {}".format(tag)) tag_event(misp, event, tag) result_event['tags'].append(tag) if 'references' in pulse: for reference in pulse['references']: if reference: log.info("\t - Adding external analysis link: {}".format(reference)) if misp: misp.add_named_attribute(event, 'link', reference, category='External analysis') result_event['attributes']['references'].append(reference) if misp and 'description' in pulse and isinstance(pulse['description'], six.text_type) and pulse['description']: log.info("\t - Adding external analysis comment") misp.add_named_attribute(event, 'comment', pulse['description'], category='External analysis') for ind in pulse['indicators']: ind_type = ind['type'] ind_val = ind['indicator'] ind_kwargs = {'to_ids': to_ids} if 'description' in ind and isinstance(ind['description'], six.text_type) and ind['description']: ind_kwargs['comment'] = ind['description'] if ind_type == 'FileHash-SHA256': log.info("\t - Adding SH256 hash: {}".format(ind_val)) if misp: misp.add_hashes(event, sha256=ind_val, **ind_kwargs) result_event['attributes']['hashes']['sha256'].append(ind_val) elif ind_type == 'FileHash-SHA1': log.info("\t - Adding SHA1 hash: {}".format(ind_val)) if misp: misp.add_hashes(event, sha1=ind_val, **ind_kwargs) result_event['attributes']['hashes']['sha1'].append(ind_val) elif ind_type == 'FileHash-MD5': log.info("\t - Adding MD5 hash: {}".format(ind_val)) if misp: misp.add_hashes(event, md5=ind_val, **ind_kwargs) result_event['attributes']['hashes']['md5'].append(ind_val) elif ind_type == 'URI' or ind_type == 'URL': log.info("\t - Adding URL: {}".format(ind_val)) if misp: misp.add_url(event, ind_val, **ind_kwargs) result_event['attributes']['urls'].append(ind_val) elif ind_type == 'domain': log.info("\t - Adding domain: {}".format(ind_val)) if misp: misp.add_domain(event, ind_val, **ind_kwargs) result_event['attributes']['domains'].append(ind_val) elif ind_type == 'hostname': log.info("\t - Adding hostname: {}".format(ind_val)) if misp: misp.add_hostname(event, ind_val, **ind_kwargs) result_event['attributes']['hostnames'].append(ind_val) elif ind_type == 'IPv4' or ind_type == 'IPv6': log.info("\t - Adding ip: {}".format(ind_val)) if misp: misp.add_ipdst(event, ind_val, **ind_kwargs) result_event['attributes']['ips'].append(ind_val) elif ind_type == 'email': log.info("\t - Adding email: {}".format(ind_val)) if misp: misp.add_email_dst(event, ind_val, **ind_kwargs) result_event['attributes']['emails'].append(ind_val) elif ind_type == 'Mutex': log.info("\t - Adding mutex: {}".format(ind_val)) if misp: misp.add_mutex(event, ind_val, **ind_kwargs) result_event['attributes']['mutexes'].append(ind_val) elif ind_type == 'CVE': log.info("\t - Adding CVE: {}".format(ind_val)) if misp: misp.add_named_attribute(event, 'vulnerability', ind_val, category='External analysis', **ind_kwargs) result_event['attributes']['cves'].append(ind_val) elif ind_type == 'FileHash-IMPHASH': log.info("\t - Adding IMPHASH hash: {}".format(ind_val)) if misp: misp.add_named_attribute(event, 'imphash', ind_val, category='Artifacts dropped', **ind_kwargs) result_event['attributes']['hashes']['imphash'].append(ind_val) elif ind_type == 'FileHash-PEHASH': log.info("\t - Adding PEHASH hash: {}".format(ind_val)) if misp: misp.add_named_attribute(event, 'pehash', ind_val, category='Artifacts dropped', **ind_kwargs) result_event['attributes']['hashes']['pehash'].append(ind_val) else: log.warning("Unsupported indicator type: %s" % ind_type) if misp and publish: event['Event']['published'] = False misp.publish(event) return result_event
def __init__(self, stream: Iterable[bytes]): self._stream = stream self._is_stream_consumed = False self._is_generator = inspect.isgenerator(stream)
def opt_generator_param(obj, param_name): if obj is not None and not inspect.isgenerator(obj): raise ParameterCheckError(( 'Param "{name}" is not a generator (return value of function that yields) Got ' "{obj} instead").format(name=param_name, obj=obj)) return obj
def complex_to_cloth(self, ctx, cls, inst, cloth, parent, name=None, as_attr=False, **kwargs): fti = cls.get_flat_type_info(cls) cls_attrs = self.get_cls_attrs(cls) # It's actually an odict but that's irrelevant here. fti_check = dict(fti.items()) elt_check = set() attrib = self._gen_attrib_dict(inst, fti) self._enter_cloth(ctx, cloth, parent, attrib=attrib, method=cls_attrs.method) for elt in self._get_elts(cloth, self.MRPC_ID): self._actions_to_cloth(ctx, cls, inst, elt) if self._is_tagbag(cloth): logger_c.debug("%r(%r) IS a tagbag", cloth, cloth.attrib) elts = self._get_elts(cloth) else: logger_c.debug("%r(%r) is NOT a tagbag", cloth, cloth.attrib) elts = self._get_outmost_elts(cloth) # Check for xmldata after entering the cloth. as_data_field = cloth.attrib.get(self.DATA_ATTR_NAME, None) if as_data_field is not None: self._process_field(ctx, cls, inst, parent, cloth, fti, as_data_field, as_attr, True, fti_check, elt_check, **kwargs) for elt in elts: for k_attr, as_attr, as_data in ((self.ID_ATTR_NAME, False, False), (self.ATTR_ATTR_NAME, True, False), (self.DATA_ATTR_NAME, False, True)): field_name = elt.attrib.get(k_attr, None) if field_name is None: continue if elt.tag == self.DATA_TAG_NAME: as_data = True ret = self._process_field(ctx, cls, inst, parent, elt, fti, field_name, as_attr=as_attr, as_data=as_data, fti_check=fti_check, elt_check=elt_check, **kwargs) if isgenerator(ret): try: while True: sv2 = (yield) ret.send(sv2) except Break as e: try: ret.throw(e) except StopIteration: pass finally: # cf below if not (as_attr or as_data): break else: # this is here so that attribute on complex model doesn't get # mixed with in-line attr inside complex model. if an element # has spyne-id, all other attrs are ignored and are processed # by the object's serializer not its parent. if not (as_attr or as_data): break if len(fti_check) > 0: logger_s.debug("No element found for the following fields: %r", list(fti_check.keys())) if len(elt_check) > 0: logger_s.debug( "No field found for element the following " "elements: %r", list(elt_check))
def test_stream(self): self.assertTrue(inspect.isgenerator(serialize.csv([])))
def _step(self, value=None, exc=None): assert not self.done(), \ '_step(): already done: {!r}, {!r}, {!r}'.format(self, value, exc) if self._must_cancel: if not isinstance(exc, futures.CancelledError): exc = futures.CancelledError() self._must_cancel = False coro = self._coro self._fut_waiter = None self.__class__._current_tasks[self._loop] = self # Call either coro.throw(exc) or coro.send(value). try: if exc is not None: result = coro.throw(exc) elif value is not None: result = coro.send(value) else: result = next(coro) except StopIteration as exc: self.set_result(exc.value) except futures.CancelledError as exc: super().cancel() # I.e., Future.cancel(self). except Exception as exc: self.set_exception(exc) except BaseException as exc: self.set_exception(exc) raise else: if isinstance(result, futures.Future): # Yielded Future must come from Future.__iter__(). if result._blocking: result._blocking = False result.add_done_callback(self._wakeup) self._fut_waiter = result if self._must_cancel: if self._fut_waiter.cancel(): self._must_cancel = False else: self._loop.call_soon( self._step, None, RuntimeError('yield was used instead of yield from ' 'in task {!r} with {!r}'.format( self, result))) elif result is None: # Bare yield relinquishes control for one event loop iteration. self._loop.call_soon(self._step) elif inspect.isgenerator(result): # Yielding a generator is just wrong. self._loop.call_soon( self._step, None, RuntimeError('yield was used instead of yield from for ' 'generator in task {!r} with {}'.format( self, result))) else: # Yielding something else is an error. self._loop.call_soon( self._step, None, RuntimeError('Task got bad yield: {!r}'.format(result))) finally: self.__class__._current_tasks.pop(self._loop) self = None # Needed to break cycles when an exception occurs.
def iscoroutine(obj): """Return True if obj is a coroutine object.""" return isinstance(obj, CoroWrapper) or inspect.isgenerator(obj)
def coro(*args, **kw): res = func(*args, **kw) if isinstance(res, futures.Future) or inspect.isgenerator(res): res = yield from res return res
params = sig.parameters # 有序字典 print(params) print('~' * 30) print(1, sig.parameters['x']) print(sig.parameters['x'].annotation) # 关键字为x 的注解信息 print(2, sig.parameters['y']) print(sig.parameters['y'].annotation) print(3, sig.parameters['args']) print(sig.parameters['args'].annotation) print(4, sig.parameters['kwargs']) print(sig.parameters['kwargs'].annotation) # inspect模块 inspect.isfunction(object) # 是否是函数 inspect.ismethod(object) # 是否是类方法 inspect.isgenerator(object) # 是否是生成器对象 inspect.isgeneratorfunction(object) # 是否是生成器函数 inspect.isclass(object) # 是否是类 inspect.ismodule(inspect) # 是否是模块 inspect.isbulitin(print) # 是否是内建对象 # parameter对象,不同于parameters # 保存在元组中,是只读的 # name,annotation,default,empty(特殊的类,用来标记default属性或注释annotation属性的空值) # kind, 实参如何绑定到形参,就是形参的类型,包括: # POSITIONAL_ONLY 值必须是位置参数提供 # POSITIONAL_OR_KEYWORD 值可以是位置参数也可以是关键字参数 # VAR_POSITIONAL 可变位置参数 # VAR_KEYWORD 可变关键字参 # KEYWORD_ONLY keyword-only参数
def subserialize(self, ctx, cls, inst, parent, ns=None, name=None): attrs = {} if self.table_name_attr is not None: attrs[self.table_name_attr] = name locale = ctx.locale with parent.element('table', attrs): fti = None if issubclass(cls, ComplexModelBase): fti = cls.get_flat_type_info(cls) if self.produce_header: with parent.element('thead'): header_row = E.tr() th = {} if self.header_cell_class is not None: th['class'] = self.header_cell_class # fti is none when the type inside Array is not a ComplexModel. if fti is None: if self.field_name_attr is not None: th[self.field_name_attr] = name header_name = self.translate(cls, ctx.locale, name) header_row.append(E.th(header_name, **th)) else: if self.field_name_attr is None: for k, v in fti.items(): header_name = self.translate(v, ctx.locale, k) header_row.append(E.th(header_name, **th)) else: for k, v in fti.items(): th[self.field_name_attr] = k header_name = self.translate(v, ctx.locale, k) header_row.append(E.th(header_name, **th)) parent.write(header_row) with parent.element('tbody'): if cls.Attributes.max_occurs > 1: ret = self.array_to_parent(ctx, cls, inst, parent, name) if isgenerator(ret): try: while True: y = (yield) ret.send(y) except Break as b: try: ret.throw(b) except StopIteration: pass else: with parent.element('tr'): ret = self.to_parent(ctx, cls, inst, parent, name) if isgenerator(ret): try: while True: y = (yield) ret.send(y) except Break as b: try: ret.throw(b) except StopIteration: pass