def recv(self, *args, **kwargs): try: data = self.connection.recv(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): return b"" else: raise SocketError(str(e)) except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b"" else: raise except OpenSSL.SSL.WantReadError: if not util.wait_for_read(self.socket, self.socket.gettimeout()): raise timeout("The read operation timed out") else: return self.recv(*args, **kwargs) # TLS 1.3 post-handshake authentication except OpenSSL.SSL.Error as e: raise ssl.SSLError("read error: %r" % e) else: return data
if count == self._ooo_server_retry or len(retry_server_list) == 0: break count += 1 serverproxy_list = retry_server_list # Check error type # Return only one error result for compability if len(result_error_set_list): return result_error_set_list[0] if len(protocol_error_list): raise ConversionError( "Protocol error while contacting OOo conversion: " "%s" % (','.join(protocol_error_list))) if len(socket_error_list): raise SocketError("%s" % (','.join(socket_error_list))) if len(fault_error_list): raise fault_error_list[0] def __getattr__(self, attr): return partial(self._proxy_function, attr) from erp5.component.mixin.DocumentExtensibleTraversableMixin import DocumentExtensibleTraversableMixin from erp5.component.interface.IConvertable import IConvertable from erp5.component.interface.ITextConvertable import ITextConvertable from erp5.component.interface.IHtmlConvertable import IHtmlConvertable from erp5.component.interface.ICachedConvertable import ICachedConvertable from erp5.component.interface.IVersionable import IVersionable from erp5.component.interface.IDownloadable import IDownloadable from erp5.component.interface.ICrawlable import ICrawlable
def run(fedora_url, remote_file_downloader, work_item_client, results_destination=None): output_file = open( os.path.join( results_destination if (results_destination and os.path.exists(results_destination)) else ".", "experiment1_{}_results.csv".format(datetime.date.today())), "a") url_file = open("fedoraurls.txt", "a") progress = [] start = str(datetime.datetime.now()) tic = time.time() while True: # obtain work item from work_item_client (see commons.py for implementations) work_item = work_item_client.get_work_item() if not work_item: break file_name = work_item.strip() actual_file_name = "2015" + file_name[4:] # download remote file from remote storage (see commons.py for implementations) download = time.time() remote_file_downloader.download_from_storage(actual_file_name, file_name) progress.append("Download," + file_name + "," + str(download) + "," + str(time.time())) # read hdf5 file f = h5py.File(file_name, 'r') processing = time.time() if f.keys()[0] is not None: data_sets = f[f.keys()[0]] channel_str = "" c = 0 for channel in data_sets.keys(): if c == len(data_sets.keys()) - 1: channel_str = channel_str + '<> dc:coverage "' + channel + '" ' else: channel_str = channel_str + '<> dc:coverage "' + channel + '" . ' c += 1 # run fits program call("fits-0.9.0/fits.sh -i " + file_name + " > " + file_name + "_fits.xml", shell=True) # read fits xml fits_xml = open(file_name + "_fits.xml", 'r').read() result = xmltodict.parse(fits_xml) description = result['fits']['identification']['identity'][0][ '@format'] format = result['fits']['identification']['identity'][0][ '@mimetype'] fits_str = '<> dc:description "' + description + '" . ' + '<> dc:format "' + format + '" . ' # create Fedora object rdf_data = 'PREFIX dc: <http://purl.org/dc/elements/1.1/> <> dc:title "' + file_name + '" . ' + \ fits_str + channel_str progress.append("Processing," + file_name + "," + str(processing) + "," + str(time.time())) ingestion = time.time() object_url = "" for x in xrange(0, 5): try: object_url = create_fedora_object(rdf_data, fedora_url, file_name) break except SocketError as e: if x == 4: raise SocketError( "retry 5 times still failed in fedora" + str(e.errno)) pass # create Fedora binary if len(object_url) > 0: file_url = object_url + "/h5" create_fedora_binary(file_name, file_url) progress.append("Ingestion," + file_name + "," + str(ingestion) + "," + str(time.time())) url_file.write(object_url + "\n") print file_url f.close() os.remove(file_name) os.remove(file_name + "_fits.xml") duration = str(time.time() - tic) end = str(datetime.datetime.now()) print duration progress.insert(0, "OVERALL EXECUTION," + start + "," + duration + "," + end) for line in progress: output_file.write(line + "\n") output_file.close() url_file.close()
def run(self): if self.lens.role == "UAC": while True: self.new_iteration() connection = Client(self.host, self.port, self.timeout, self.bsize) # generate first client message and send to the server snd_message = self.lens.transitionSelf() print "\n>>> SENDING msg:\n{}".format(snd_message[1]) try: connection.send(str(snd_message[1])) except socket.error as e: print "socket: {} in send operation".format(e) while self.status == FUZZ_STATUS_OK: #time.sleep(0.3) try: rcv_message = connection.recv() if len(rcv_message) == 0: e = SocketError("Received empty message") e.errno = 0 raise e deco = '#' * 80 quoted_rcv_msg = urllib.quote(rcv_message) print ">>> RECEIVED message:\n{}\n{}\n{}".format( deco, quoted_rcv_msg, deco) except Exception as e: print "socket: {}\n".format(e) self._termination_check() if self.status is FUZZ_STATUS_TERMINATED: connection.close() self._crash_check() self.log_trace() break else: if e.errno == errno.ECONNRESET: connection.close() connection.accept() continue if e.errno == errno.ETIMEDOUT: pass print ">>> Consuming RECEIVED msg of length {}".format( len(rcv_message)) status = self.lens.consumeOtherSide(rcv_message) print ">>> STATUS: {}".format(status) if status == "END": self.new_iteration(reset=1) status = self.lens.consumeOtherSide(rcv_message) print ">>> STATUS: {}".format(status) if status == "NO TRANSITION": print "Consuming empty message..." status = self.lens.consumeOtherSide("") self.log_trace() status, msg, transition = self.lens.transitionSelf() print ">>> STATUS: {}, TRANSITION: {}".format( status, transition) if msg is not None: try: connection.send(str(msg)) print ">>> SENDING msg of length {}".format( len(msg)) if len(msg) < 5000: deco = '#' * 80 print "{}\n{}\n{}".format(deco, msg, deco) except socket.error as e: print "socket: {} in send operation".format(e) connection.close() break elif self.lens.role == "UAS": connection = Server(self.host, self.port, self.timeout, self.bsize) while True: if self.cuckoo_session: self._run_sample() connection.accept() connection.settimeout(self.timeout) self.new_iteration() rcv_message = "" while self.status == FUZZ_STATUS_OK: try: print ">>> RECEIVING message... " rcv_message = connection.recv() if len(rcv_message) == 0: e = SocketError("Received empty message") e.errno = 0 raise e deco = '#' * 80 quoted_rcv_msg = urllib.quote(rcv_message) print ">>> RECEIVED message:\n{}\n{}\n{}".format( deco, quoted_rcv_msg, deco) except Exception as e: print "socket: {}".format(e) self._termination_check() if self.status is FUZZ_STATUS_TERMINATED: connection.close() self._crash_check() self.log_trace() break else: if e.errno == errno.ECONNRESET: connection.close() connection.accept() continue if e.errno == errno.ETIMEDOUT: pass #if rcv_message != "": print ">>> Consuming RECEIVED msg of length {}".format( len(rcv_message)) status = self.lens.consumeOtherSide(rcv_message) print ">>> STATUS: {}".format(status) if status == "END": self.new_iteration(reset=1) status = self.lens.consumeOtherSide(rcv_message) print ">>> STATUS: {}".format(status) if status == "NO TRANSITION": print "Consuming empty message..." status = self.lens.consumeOtherSide("") self.log_trace() status, msg, transition = self.lens.transitionSelf() print ">>> STATUS: {}, TRANSITION: {}".format( status, transition) if msg is not None: try: connection.send(str(msg)) print ">>> SENDING msg of length {}".format( len(msg)) if len(msg) < 5000: deco = '#' * 80 print "{}\n{}\n{}".format(deco, msg, deco) except socket.error as e: print "socket: {} in send operation".format(e) connection.close() break
class NetlinkMixin(object): ''' Generic netlink socket ''' def __init__(self, family=NETLINK_GENERIC, port=None, pid=None): super(NetlinkMixin, self).__init__(AF_NETLINK, SOCK_DGRAM, family) global sockets self.recv_plugin = self.recv_plugin_init # 8<----------------------------------------- # PID init is here only for compatibility, # later it will be completely moved to bind() self.addr_pool = AddrPool(minaddr=0xff) self.epid = None self.port = 0 self.fixed = True self.backlog = {0: []} self.monitor = False self.callbacks = [] # [(predicate, callback, args), ...] self.clean_cbs = {} # {msg_seq: [callback, ...], ...} self.pthread = None self.backlog_lock = threading.Lock() self.read_lock = threading.Lock() self.change_master = threading.Event() self.lock = LockFactory() self.buffer_queue = Queue() self.qsize = 0 self.log = [] self.get_timeout = 3 self.get_timeout_exception = None if pid is None: self.pid = os.getpid() & 0x3fffff self.port = port self.fixed = self.port is not None elif pid == 0: self.pid = os.getpid() else: self.pid = pid # 8<----------------------------------------- self.groups = 0 self.marshal = Marshal() # 8<----------------------------------------- # Set default sockopts self.setsockopt(SOL_SOCKET, SO_SNDBUF, 32768) self.setsockopt(SOL_SOCKET, SO_RCVBUF, 1024 * 1024) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self.close() def release(self): logging.warning("The `release()` call is deprecated") logging.warning("Use `close()` instead") self.close() def register_callback(self, callback, predicate=lambda x: True, args=None): ''' Register a callback to run on a message arrival. Callback is the function that will be called with the message as the first argument. Predicate is the optional callable object, that returns True or False. Upon True, the callback will be called. Upon False it will not. Args is a list or tuple of arguments. Simplest example, assume ipr is the IPRoute() instance:: # create a simplest callback that will print messages def cb(msg): print(msg) # register callback for any message: ipr.register_callback(cb) More complex example, with filtering:: # Set object's attribute after the message key def cb(msg, obj): obj.some_attr = msg["some key"] # Register the callback only for the loopback device, index 1: ipr.register_callback(cb, lambda x: x.get('index', None) == 1, (self, )) Please note: you do **not** need to register the default 0 queue to invoke callbacks on broadcast messages. Callbacks are iterated **before** messages get enqueued. ''' if args is None: args = [] self.callbacks.append((predicate, callback, args)) def unregister_callback(self, callback): ''' Remove the first reference to the function from the callback register ''' cb = tuple(self.callbacks) for cr in cb: if cr[1] == callback: self.callbacks.pop(cb.index(cr)) return def register_policy(self, policy, msg_class=None): ''' Register netlink encoding/decoding policy. Can be specified in two ways: `nlsocket.register_policy(MSG_ID, msg_class)` to register one particular rule, or `nlsocket.register_policy({MSG_ID1: msg_class})` to register several rules at once. E.g.:: policy = {RTM_NEWLINK: ifinfmsg, RTM_DELLINK: ifinfmsg, RTM_NEWADDR: ifaddrmsg, RTM_DELADDR: ifaddrmsg} nlsocket.register_policy(policy) One can call `register_policy()` as many times, as one want to -- it will just extend the current policy scheme, not replace it. ''' if isinstance(policy, int) and msg_class is not None: policy = {policy: msg_class} assert isinstance(policy, dict) for key in policy: self.marshal.msg_map[key] = policy[key] return self.marshal.msg_map def unregister_policy(self, policy): ''' Unregister policy. Policy can be: * int -- then it will just remove one policy * list or tuple of ints -- remove all given * dict -- remove policies by keys from dict In the last case the routine will ignore dict values, it is implemented so just to make it compatible with `get_policy_map()` return value. ''' if isinstance(policy, int): policy = [policy] elif isinstance(policy, dict): policy = list(policy) assert isinstance(policy, (tuple, list, set)) for key in policy: del self.marshal.msg_map[key] return self.marshal.msg_map def get_policy_map(self, policy=None): ''' Return policy for a given message type or for all message types. Policy parameter can be either int, or a list of ints. Always return dictionary. ''' if policy is None: return self.marshal.msg_map if isinstance(policy, int): policy = [policy] assert isinstance(policy, (list, tuple, set)) ret = {} for key in policy: ret[key] = self.marshal.msg_map[key] return ret def bind(self, groups=0, pid=None, async=False): ''' Bind the socket to given multicast groups, using given pid. * If pid is None, use automatic port allocation * If pid == 0, use process' pid * If pid == <int>, use the value instead of pid ''' if pid is not None: self.port = 0 self.fixed = True self.pid = pid or os.getpid() self.groups = groups # if we have pre-defined port, use it strictly if self.fixed: self.epid = self.pid + (self.port << 22) super(NetlinkMixin, self).bind((self.epid, self.groups)) else: # if we have no pre-defined port, scan all the # range till the first available port for i in range(1024): try: self.port = sockets.alloc() self.epid = self.pid + (self.port << 22) super(NetlinkMixin, self).bind((self.epid, self.groups)) # if we're here, bind() done successfully, just exit break except SocketError as e: # pass occupied sockets, raise other exceptions if e.errno != 98: raise else: # raise "address in use" -- to be compatible raise SocketError(98, 'Address already in use') # all is OK till now, so start async recv, if we need if async: self._stop = False self.recv_plugin = self.recv_plugin_queue self.pthread = threading.Thread(target=self.async_recv) self.pthread.setDaemon(True) self.pthread.start()
def getpeername(self): if self.test.peerNameSucceed: return ("4.3.2.1", 4321) else: raise SocketError(ENOTCONN, "Transport endpoint not connected")
def _connect(self, url, params, ok_codes, rtype, description, redirect=False): """Handle fetching, logging, errors and history.""" if params is None and rtype in ('post', 'put'): # enable empty put/post params = [] t_start = time.time() try: response = self._browser.fetch(url, params, ok_codes=ok_codes, key_file=self._keyfile_path, cert_file=self._certfile_path, method=rtype) except: etype, value, tback = sys.exc_info() t_stop = time.time() t_delta = t_stop - t_start self.total_time += t_delta self.step_success = False self.test_status = 'Failure' self.logd(' Failed in %.3fs' % t_delta) if etype is HTTPError: self._log_response(value.response, rtype, description, t_start, t_stop, log_body=True) if self._dumping: self._dump_content(value.response) raise self.failureException, str(value.response) else: self._log_response_error(url, rtype, description, t_start, t_stop) if etype is SocketError: raise SocketError("Can't load %s." % url) raise t_stop = time.time() # Log response t_delta = t_stop - t_start self.total_time += t_delta if redirect: self.total_redirects += 1 elif rtype != 'link': self.total_pages += 1 else: self.total_links += 1 if rtype in ('put', 'post', 'get', 'delete'): # this is a valid referer for the next request self.setHeader('Referer', url) self._browser.history.append((rtype, url)) self.logd(' Done in %.3fs' % t_delta) self._log_response(response, rtype, description, t_start, t_stop) if self._dumping: self._dump_content(response) return response
def sendall(self, data, flags=None): log.info("Failing send") raise SocketError()
class TestConfig(unittest.TestCase): fsock = None mockdate = "2017-09-06 00:44:47.7" def _redirectOut(self): # pragma: no cover self.fsock = open('out.log', 'a+') sys.stdout = self.fsock def tearDown(self): # pragma: no cover if self.fsock is not None: self.fsock.close() self.fsock = None sys.stdout = sys.__stdout__ def setUp(self): self.config = Config() self.configXMLString = XMLString self.config.parseFromString(self.configXMLString) self.configFBXML = FBXML def test_timestamp(self): """ test getlogtime :return: """ stamp = helpers.getLoggingTime() self.assertRegexpMatches(stamp, "\d{4}-\d{2}-\d{2}", "should have time stamp,\n" + stamp) def test_createdir(self): """ test dir with file :return: """ helpers.createResultsDir("none") self.assertEqual(os.path.isdir(helpers.RESULTS_DIR), True, "missing directory " + helpers.RESULTS_DIR) @patch('os.path.dirname', new=Mock(side_effect=OSError("fail to mock a write", errno.EACCES))) def test_createdir_raise(self): """ test dir raise error :return: """ self.assertRaisesRegexp(OSError, "fail to mock", helpers.createResultsDir, "none") @patch('os.path.dirname', new=Mock(side_effect=OSError("fail to mock a write", errno.EEXIST))) def test_createdir_fail(self): """ test dir with failing oserror :return: """ self.assertRaisesRegexp(OSError, "fail to mock", helpers.createResultsDir, "none") def test_dump_none(self): """ test none page to a file :return: """ self.assertRaisesRegexp(TypeError, "None", helpers.dumpErrorPage, None) def test_dump(self): """ test dump page to a file :return: """ filename = helpers.dumpErrorPage(self.mockdate) output = "" with open("result/" + filename, "r") as fd: output += fd.readline() self.assertRegexpMatches(output, "\d{4}-\d{2}-\d{2}", "should have time stamp,\n" + output) def test_errorontext(self): """ test exception from helper's errorOnText :return: """ err = 'Authentication has not been passed: Invalid password' # not found so no assertion output = helpers.errorOnText("", 'That password is incorrect.', err) # should raise if it sees an assertion self.assertRaisesRegexp(helpers.BingAccountError, "Invalid", helpers.errorOnText, 'That password is incorrect.', 'That password is incorrect.', err) def test_node(self): """ test node's children :return: """ import xml.etree.ElementTree as ET root = ET.fromstring(self.configXMLString) node = helpers.getXmlChildNodes(root) self.assertIsNotNone(node, "should not be null " + str(node)) @patch('sys.version_info') def test_node_fail(self, mockver): sys.version_info = [2, 1] import xml.etree.ElementTree as ET root = ET.fromstring(self.configXMLString) node = helpers.getXmlChildNodes(root) self.assertIsNotNone(node, "should not be null " + str(node)) def test_accounts(self): self.assertIsNotNone(self.config.accounts) self.assertEqual(len(self.config.accounts), 1) accounts = dict() acc = Config.Account() acc.accountLogin = "******" acc.password = "******" acc.accountType = "Live" acc.disabled = False acc.ua_desktop = "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136" acc.ua_mobile = "mozilla/5.0 (iphone; cpu iphone os 7_0_2 like mac os x) applewebkit/537.51.1 (khtml, like gecko) version/7.0 mobile/11a501 safari/9537.53" accounts[acc.getRef()] = acc self.assertEqual(accounts, self.config.accounts) def test_history(self): """ test history parsing :return: """ self.assertRaisesRegexp(TypeError, "None", bingHistory.parse, None) output = bingHistory.parse("") self.assertIsNotNone(output, "missing output " + str(output)) page = '<span class="query_t">' page += '<div id="results_area"></div><div id="sidebar"></div>' output = bingHistory.parse(page) self.assertIsNotNone(output, "missing output " + str(output)) page = '<span class="sh_item_qu_query">' page += '<ul class="sh_dayul"></ul>' page += ' value == 0' page += '</span>' output = bingHistory.parse(page) self.assertIsNotNone(output, "missing output " + str(output)) page = '<ul class="sh_dayul"> </ul>' output = bingHistory.parse(page) self.assertIsNotNone(output, "missing output " + str(output)) output = bingHistory.getBingHistoryTodayURL() self.assertRegexpMatches(output, "https", "missing url " + str(output)) @patch('helpers.getResponseBody', return_value='"WindowsLiveId":"" "WindowsLiveId":""') @patch('time.sleep', return_value='') def test_auth_url(self, timemock, helpmock): # pragma: no cover """ test authentication decoding error :return: """ self.assertRaisesRegexp(ValueError, "unknown url type", run, self.config) @patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=SocketError(errno.ECONNREFUSED, "errno.ECONNREFUSED"))) def test_auth_exceptionSock(self): self.assertRaisesRegexp(SocketError, "", run, self.config) @patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=SocketError(errno.ECONNRESET, "errno.ECONNRESET"))) def test_auth_exceptionSockReset(self): self.assertIsNone(run(self.config), "should not return anything") @patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=helpers.BingAccountError(None))) def test_auth_exceptionBing(self): self.assertIsNone(run(self.config), "should not return anything") @patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=urllib2.URLError(""))) def test_auth_exceptionURL(self): self.assertIsNone(run(self.config), "should not return anything") @patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=HTMLParser.HTMLParseError("error"))) def test_auth_exceptionParser(self): self.assertIsNone(run(self.config), "should not return anything") @patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=urllib2.HTTPError("", "", "", "", open("tmp", "a+")))) def test_auth_exceptionHTTP(self): self.assertIsNone(run(self.config), "should not return anything") def test_stringify(self): self.assertRaisesRegexp(ValueError, "too small", stringify, None, -1) @patch('urllib2.Request', return_value="") @patch('helpers.getResponseBody', return_value="") @patch('urllib2.Request.add_header', return_value=urllib2.Request(bingCommon.BING_URL, bingCommon.HEADERS)) def test_auth_url(self, headermock, helpmock, urlmock): """ test auth class :param headermock: :param helpmock: :param urlmock: :return: """ self.assertRaisesRegexp(TypeError, "opener is not", bingAuth.BingAuth, bingCommon.HEADERS, None) auth = bingAuth.BingAuth(bingCommon.HEADERS, urllib2.OpenerDirector()) self.assertIsNotNone(auth, "should return class") def test_config(self): """ test config module :return: """ configobj = Config() self.assertIsNotNone(configobj, "should return class") self.assertIsNotNone(Config.General(), "should return class") self.assertIsNotNone(ConfigError("ok"), "should return exception") self.assertIsNotNone(Config.Proxy(), "should return class") self.assertIsNotNone(Config.EventAccount(), "should return class") self.assertIsNotNone(Config.Event.Notify(), "should return class") ifs = Config.Event.IfStatement() ifs.op = lambda x, y: x ifs.lhs = lambda x: x ifs.rhs = "b" self.assertIsNotNone(str(ifs), "should return class") self.assertRaisesRegexp(ValueError, "None", ifs.evaluate, None) self.assertRaisesRegexp(TypeError, "is not of", ifs.evaluate, []) self.assertIsNotNone(ifs.evaluate(BingRewardsReportItem())) spec = Config.Event.Specifier() self.assertIsNotNone(spec, "should return class") self.assertRaisesRegexp(ValueError, "is None", spec.evaluate, None, BingRewardsReportItem()) self.assertRaisesRegexp(TypeError, "list", spec.evaluate, [], BingRewardsReportItem()) self.assertRaisesRegexp(ValueError, "is None", spec.evaluate, [], None) self.assertRaisesRegexp(TypeError, "not of BingRewardsReportItem type", spec.evaluate, [], self.config) self.assertIsNotNone(spec.evaluate("%a", BingRewardsReportItem()), "should return string") dist = os.path.join(os.path.dirname(__file__), "..", "config.xml") self.assertIsNone(configobj.parseFromFile(dist), "should be none") self.assertRaisesRegexp(ValueError, "_configFile_ is None", configobj.parseFromFile, None) self.assertRaisesRegexp(ValueError, "is None", self.config.parseFromString, None) self.assertRaisesRegexp(ConfigError, "Invalid subnode", configobj.parseFromString, InvalidXML) self.assertRaisesRegexp(ConfigError, "is not found", configobj.parseFromString, LOGINXML) self.assertRaisesRegexp(ConfigError, "is not found", configobj.parseFromString, PWDXML) self.assertRaisesRegexp(ConfigError, "should be either set", self.config.parseFromString, PROXYLOGINXML) self.assertRaisesRegexp(KeyError, "_specifier_ is not", validateSpecifier, "%not") self.assertRaisesRegexp(ConfigError, "Invalid subnode", self.config.parseFromString, FBXML) def test_config_attr(self): self.assertRaisesRegexp(ConfigError, "MUST", self.config.parseFromString, FLOAT) self.assertRaisesRegexp(ConfigError, "MUST", self.config.parseFromString, INT) self.assertRaisesRegexp(ConfigError, "must", self.config.parseFromString, NONFLOAT) self.assertRaisesRegexp(ConfigError, "must", self.config.parseFromString, NONINT) def test_config_notify(self): self.assertRaisesRegexp(ConfigError, "is not found", self.config.parseFromString, NONREF) self.assertRaisesRegexp(ConfigError, "is not found", self.config.parseFromString, NONACCREF) self.assertRaisesRegexp(ConfigError, "not supported", self.config.parseFromString, NONEV) def test_config_retry(self): self.assertRaisesRegexp(ConfigError, "is not found", self.config.parseFromString, RETRY) self.assertRaisesRegexp(ConfigError, "must be", self.config.parseFromString, INVRETRY) self.assertRaisesRegexp(ConfigError, "MUST BE", self.config.parseFromString, NEGRETRY) self.assertRaisesRegexp(ConfigError, "is not found", self.config.parseFromString, RETRYCNT) self.assertRaisesRegexp(ConfigError, "must be", self.config.parseFromString, INVRETRYCNT) self.assertRaisesRegexp(ConfigError, "MUST BE", self.config.parseFromString, NEGRETRYCNT) def test_config_if(self): self.assertRaisesRegexp(ConfigError, "is invalid", self.config.parseFromString, NONIF2) self.assertRaisesRegexp(ConfigError, "is invalid", self.config.parseFromString, NONIFRHS) self.assertRaisesRegexp(ConfigError, "is invalid", self.config.parseFromString, NONIFOP) def test_event(self): """ test event :return: """ self.assertIsNone(EventsProcessor.onScriptFailure(self.config, Exception()), "should be none") self.assertIsNone(EventsProcessor.onScriptComplete(self.config), "should be none") self.assertRaisesRegexp(ConfigError, "not found", self.config.parseFromString, EVENT) self.config.parseFromString(EVENTLESS) self.assertRaisesRegexp(Exception, ".*", EventsProcessor.onScriptFailure, self.config, Exception()) self.assertIsNone(EventsProcessor.onScriptComplete(self.config), "should be none") ep = EventsProcessor(self.config, BingRewardsReportItem()) self.assertIsNotNone(ep.processReportItem(), "should not be none and be done") @patch('main.earnRewards', return_value=None) @patch('eventsProcessor.EventsProcessor.processReportItem', return_value=(-1, None)) def test_event_dontcare(self, mockep, mockmain): # not retry nor ok with -1 self.assertIsNone(processAccount(self.config), "should return nothing") def test_event_getEvent_returnsEvent(self): """ test onScriptFailure using echo from xml config string :return: """ event = self.config.getEvent(Config.Event.onScriptFailure) self.assertIsNotNone(event) self.assertTrue(len(event.notifies) == 1) self.assertEqual(event.notifies[0].cmd, "echo") def test_event_getEvent_returnsNoneIfEventDoesntExist(self): """ test no event call does not exist :return: """ self.assertIsNone(self.config.getEvent("does_not_exist")) self.assertRaisesRegexp(ValueError, "None", self.config.getEvent, None) def test_reward_bfp_hit(self): self._rewards_hit(bfp.RewardV1()) self._rewards_hit(bdp.Reward()) @patch('helpers.getResponseBody') def _rewards_hit(self, classobj, helpmock): """ test rewards object :return: """ self.config.proxy = False reward = BingRewards(bingCommon.HEADERS, "", self.config) page = '"WindowsLiveId":"" "WindowsLiveId":"" ' page += 'action="0" value="0" ' page += 'value= "0" NAP value="0" ' page += 'ANON value="0" ' page += 'id="t" value="0" ' page += '<div> 999 livetime points</div> ' helpmock.return_value = page # if not login should have not found error for url self.assertIsNotNone(reward.getLifetimeCredits, "Should return int") page = "t.innerHTML='100'" helpmock.return_value = page self.assertIsNotNone(reward.getRewardsPoints(), "should not be None") self.assertRaisesRegexp(TypeError, "not an instance", reward.process, None, True) # NONE case newbfp = classobj newbfp.tp = None rewards = [newbfp] self.assertIsNotNone(reward.process(rewards, True), "handle not none") # HIT case newbfp.tp = mock.Mock() newbfp.tp = [0, 1, 2, 3, bfp.RewardV1.Type.Action.HIT] # SEARCH case newbfp.tp = mock.Mock() newbfp.tp = [0, 1, 2, 3, bfp.RewardV1.Type.Action.SEARCH] newbfp.progressCurrent = 100 rewards = [newbfp] self.assertIsNotNone(reward.process(rewards, True), "should return res") self.assertRaisesRegexp(TypeError, "not an instance", reward.printResults, None, True) result = mock.Mock() result.action = bfp.RewardV1.Type.Action.SEARCH result.isError = True result.o = newbfp result.message = "done" newbfp.progressCurrent = 1 newbfp.progressMax = 100 newbfp.url = "http:0.0.0.0" self.assertIsNone(reward.printResults([result], True), "should return None") self.assertRaisesRegexp(TypeError, "rewards is not", reward.printRewards, None) rewards[0].isDone = True self.assertIsNone(reward.printRewards(rewards), "should return None") self.assertRaisesRegexp(TypeError, "reward is not", reward.RewardResult, None) self.assertIsNotNone(reward.RewardResult(newbfp), "should return class") proxy = mock.Mock() proxy.login = True proxy.password = "******" proxy.url = "http://127.0.0.1" proxy.protocols = "http" self.config.proxy = proxy self.assertIsNotNone(BingRewards(bingCommon.HEADERS, "", self.config), "should return class") proxy.login = False self.config.proxy = proxy self.assertIsNotNone(BingRewards(bingCommon.HEADERS, "", self.config), "should return class") self.assertRaisesRegexp(ConfigError, "not found", self.config.parseFromString, PROTXML) self.assertRaisesRegexp(ConfigError, "not found", self.config.parseFromString, URLXML)
def _setup_https_tunnel(self): sock = self.sock host = self._tunnel_host port = self._tunnel_port try: lines = [] lines.append('CONNECT %s:%d HTTP/1.1' % (host, port)) lines.append('Host: %s:%d' % (host, port)) if self._tunnel_headers: for item in self._tunnel_headers.items(): lines.append('%s: %s' % item) data = '\r\n'.join(lines) + '\r\n\r\n' sock.sendall(data.encode()) data = b'' code = 0 pos = -1 while True: s = sock.recv(4096) if not s: if code == 0: raise SocketError("Tunnel connection failed: %r" % data) break data += s if code == 0 and b'\r\n' in data: version, code, message = data.split(b' ', 2) if code != b'200': sock.close() raise SocketError("Tunnel connection failed: %s %s" % (code, message.strip())) pos = data.find(b'\r\n\r\n') if pos > 0: break tls_conn = tlslite.TLSConnection(sock) try: tls_conn.handshakeClientCert(serverName=host) except Exception: sock.close() raise try: ssl.match_hostname(tlslite_getpeercert(tls_conn), host) except Exception: tls_conn.close() raise except SocketTimeout as e: raise ConnectTimeoutError( self, "Connection to %s timed out. (connect timeout=%s)" % (self.host, self.timeout)) except SocketError as e: raise NewConnectionError( self, "Failed to establish a new connection: %s" % e) # patch fileno, # let urllib3.util.connection.is_connection_dropped work as expected # tls_conn.fileno = partial(self._origin_sock.fileno) # here we always got fileno = -1 tls_conn.fileno = partial(sock.fileno) # patch getpeercert tls_conn.getpeercert = partial(tlslite_getpeercert, tls_conn) self.sock = tls_conn
def sendall(self, data, flags=None): raise SocketError()
def raise_socket_error(*args, **kwargs): raise SocketError()
def send(self, data): while True: try: return self.connection.send(data) except OpenSSL.SSL.SysCallError as e: raise SocketError(str(e))
def _proxy_function(self, func_name, *args, **kw): result_error_set_list = [] protocol_error_list = [] socket_error_list = [] fault_error_list = [] count = 0 serverproxy_list = self._serverproxy_list # we have list of tuple (uri, ServerProxy()). Sort by uri having oldest failure serverproxy_list.sort( key=lambda x: global_server_proxy_uri_failure_time.get(x[0], 0)) while True: retry_server_list = [] for uri, server_proxy in serverproxy_list: func = getattr(server_proxy, func_name) failure = True try: # Cloudooo return result in (200 or 402, dict(), '') format or just based type # 402 for error and 200 for ok result_set = func(*args, **kw) except SocketError as e: message = 'Socket Error: %s' % (repr(e) or 'undefined.') socket_error_list.append(message) retry_server_list.append((uri, server_proxy)) except ProtocolError as e: # Network issue message = "%s: %s %s" % (e.url, e.errcode, e.errmsg) if e.errcode == -1: message = "%s: Connection refused" % (e.url) protocol_error_list.append(message) retry_server_list.append((uri, server_proxy)) except Fault as e: # Return not supported data types fault_error_list.append(e) else: failure = False if not (failure): try: response_code, _, _ = result_set except ValueError: # Compatibility for old oood, result is based type, like string response_code = 200 if response_code == 200: return result_set else: # If error, try next one result_error_set_list.append(result_set) # Still there ? this means we had no result, # avoid using same server again global_server_proxy_uri_failure_time[uri] = int(DateTime()) # All servers are failed if count == self._ooo_server_retry or len(retry_server_list) == 0: break count += 1 serverproxy_list = retry_server_list # Check error type # Return only one error result for compability if len(result_error_set_list): return result_error_set_list[0] if len(protocol_error_list): raise ConversionError( "Protocol error while contacting OOo conversion: " "%s" % (','.join(protocol_error_list))) if len(socket_error_list): raise SocketError("%s" % (','.join(socket_error_list))) if len(fault_error_list): raise fault_error_list[0]
def fail(message): if 'resolved' not in state: state['resolved'] = True self._cb = None router_error(SocketError(message))
# Copy-pasted from Python 3.5 source code def _decref_socketios(self): if self._makefile_refs > 0: self._makefile_refs -= 1 if self._closed: self.close() def recv(self, *args, **kwargs): try: data = self.connection.recv(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: <<<<<<< HEAD if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): return b"" else: raise SocketError(str(e)) except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b"" ======= if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): return b'' else: raise SocketError(str(e)) except OpenSSL.SSL.ZeroReturnError as e: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b'' >>>>>>> 71358189c5e72ee2ac9883b408a2f540a7f5745e else: raise except OpenSSL.SSL.WantReadError:
def run(self, text, options, path): """ Sends the TTS request to ImTranslator, captures the audio from the returned SWF, and transcodes to MP3. Because ImTranslator sometimes raises various errors, both steps of this (i.e. downloading the page and dumping the audio) may be retried up to three times. """ output_wavs = [] output_mp3s = [] require = dict(size_in=4096) logger = self._logger try: for subtext in self.util_split(text, 400): for i in range(1, 4): try: logger.info("ImTranslator net_stream: attempt %d", i) result = self.net_stream( ('http://imtranslator.net/translate-and-speak/' 'sockets/tts.asp', dict(text=subtext, vc=options['voice'], speed=options['speed'], FA=1)), require=dict(mime='text/html', size=256), method='POST', ) result = self._RE_SWF.search(result) if not result or not result.group(): raise EnvironmentError('500b', "cannot find SWF" "path in payload") result = result.group() except (EnvironmentError, IOError) as error: if getattr(error, 'code', None) == 500: logger.warn("ImTranslator net_stream: got 500") elif getattr(error, 'errno', None) == '500b': logger.warn("ImTranslator net_stream: no SWF path") elif 'timed out' in format(error): logger.warn("ImTranslator net_stream: timeout") else: logger.error("ImTranslator net_stream: %s", error) raise else: logger.info("ImTranslator net_stream: success") break else: logger.error("ImTranslator net_stream: exhausted") raise SocketError("unable to fetch page from ImTranslator " "even after multiple attempts") output_wav = self.path_temp('wav') output_wavs.append(output_wav) for i in range(1, 4): try: logger.info("ImTranslator net_dump: attempt %d", i) self.net_dump(output_wav, result) except RuntimeError: logger.warn("ImTranslator net_dump: failure") else: logger.info("ImTranslator net_dump: success") break else: logger.error("ImTranslator net_dump: exhausted") raise SocketError("unable to dump audio from ImTranslator " "even after multiple attempts") if len(output_wavs) > 1: for output_wav in output_wavs: output_mp3 = self.path_temp('mp3') output_mp3s.append(output_mp3) self.cli_transcode(output_wav, output_mp3, require=require) self.util_merge(output_mp3s, path) else: self.cli_transcode(output_wavs[0], path, require=require) finally: self.path_unlink(output_wavs, output_mp3s)
def test_open_connection_error(self, mock_connection, mock_path): # raise exception mock_connection.connect.side_effect = SocketError() with self.assertRaises(CommunicationError): self._socket.open()