def recv(self, *args, **kwargs):
try:
data = self.connection.recv(*args, **kwargs)
except OpenSSL.SSL.SysCallError as e:
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
return b""
else:
raise SocketError(str(e))
except OpenSSL.SSL.ZeroReturnError:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return b""
else:
raise
except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
raise timeout("The read operation timed out")
else:
return self.recv(*args, **kwargs)
# TLS 1.3 post-handshake authentication
except OpenSSL.SSL.Error as e:
raise ssl.SSLError("read error: %r" % e)
else:
return data
if count == self._ooo_server_retry or len(retry_server_list) == 0:
break
count += 1
serverproxy_list = retry_server_list
# Check error type
# Return only one error result for compability
if len(result_error_set_list):
return result_error_set_list[0]
if len(protocol_error_list):
raise ConversionError(
"Protocol error while contacting OOo conversion: "
"%s" % (','.join(protocol_error_list)))
if len(socket_error_list):
raise SocketError("%s" % (','.join(socket_error_list)))
if len(fault_error_list):
raise fault_error_list[0]
def __getattr__(self, attr):
return partial(self._proxy_function, attr)
from erp5.component.mixin.DocumentExtensibleTraversableMixin import DocumentExtensibleTraversableMixin
from erp5.component.interface.IConvertable import IConvertable
from erp5.component.interface.ITextConvertable import ITextConvertable
from erp5.component.interface.IHtmlConvertable import IHtmlConvertable
from erp5.component.interface.ICachedConvertable import ICachedConvertable
from erp5.component.interface.IVersionable import IVersionable
from erp5.component.interface.IDownloadable import IDownloadable
from erp5.component.interface.ICrawlable import ICrawlable
def run(fedora_url,
remote_file_downloader,
work_item_client,
results_destination=None):
output_file = open(
os.path.join(
results_destination if
(results_destination and os.path.exists(results_destination)) else
".", "experiment1_{}_results.csv".format(datetime.date.today())),
"a")
url_file = open("fedoraurls.txt", "a")
progress = []
start = str(datetime.datetime.now())
tic = time.time()
while True:
# obtain work item from work_item_client (see commons.py for implementations)
work_item = work_item_client.get_work_item()
if not work_item:
break
file_name = work_item.strip()
actual_file_name = "2015" + file_name[4:]
# download remote file from remote storage (see commons.py for implementations)
download = time.time()
remote_file_downloader.download_from_storage(actual_file_name,
file_name)
progress.append("Download," + file_name + "," + str(download) + "," +
str(time.time()))
# read hdf5 file
f = h5py.File(file_name, 'r')
processing = time.time()
if f.keys()[0] is not None:
data_sets = f[f.keys()[0]]
channel_str = ""
c = 0
for channel in data_sets.keys():
if c == len(data_sets.keys()) - 1:
channel_str = channel_str + '<> dc:coverage "' + channel + '" '
else:
channel_str = channel_str + '<> dc:coverage "' + channel + '" . '
c += 1
# run fits program
call("fits-0.9.0/fits.sh -i " + file_name + " > " + file_name +
"_fits.xml",
shell=True)
# read fits xml
fits_xml = open(file_name + "_fits.xml", 'r').read()
result = xmltodict.parse(fits_xml)
description = result['fits']['identification']['identity'][0][
'@format']
format = result['fits']['identification']['identity'][0][
'@mimetype']
fits_str = '<> dc:description "' + description + '" . ' + '<> dc:format "' + format + '" . '
# create Fedora object
rdf_data = 'PREFIX dc: <http://purl.org/dc/elements/1.1/> <> dc:title "' + file_name + '" . ' + \
fits_str + channel_str
progress.append("Processing," + file_name + "," + str(processing) +
"," + str(time.time()))
ingestion = time.time()
object_url = ""
for x in xrange(0, 5):
try:
object_url = create_fedora_object(rdf_data, fedora_url,
file_name)
break
except SocketError as e:
if x == 4:
raise SocketError(
"retry 5 times still failed in fedora" +
str(e.errno))
pass
# create Fedora binary
if len(object_url) > 0:
file_url = object_url + "/h5"
create_fedora_binary(file_name, file_url)
progress.append("Ingestion," + file_name + "," +
str(ingestion) + "," + str(time.time()))
url_file.write(object_url + "\n")
print file_url
f.close()
os.remove(file_name)
os.remove(file_name + "_fits.xml")
duration = str(time.time() - tic)
end = str(datetime.datetime.now())
print duration
progress.insert(0,
"OVERALL EXECUTION," + start + "," + duration + "," + end)
for line in progress:
output_file.write(line + "\n")
output_file.close()
url_file.close()
def run(self):
if self.lens.role == "UAC":
while True:
self.new_iteration()
connection = Client(self.host, self.port, self.timeout,
self.bsize)
# generate first client message and send to the server
snd_message = self.lens.transitionSelf()
print "\n>>> SENDING msg:\n{}".format(snd_message[1])
try:
connection.send(str(snd_message[1]))
except socket.error as e:
print "socket: {} in send operation".format(e)
while self.status == FUZZ_STATUS_OK:
#time.sleep(0.3)
try:
rcv_message = connection.recv()
if len(rcv_message) == 0:
e = SocketError("Received empty message")
e.errno = 0
raise e
deco = '#' * 80
quoted_rcv_msg = urllib.quote(rcv_message)
print ">>> RECEIVED message:\n{}\n{}\n{}".format(
deco, quoted_rcv_msg, deco)
except Exception as e:
print "socket: {}\n".format(e)
self._termination_check()
if self.status is FUZZ_STATUS_TERMINATED:
connection.close()
self._crash_check()
self.log_trace()
break
else:
if e.errno == errno.ECONNRESET:
connection.close()
connection.accept()
continue
if e.errno == errno.ETIMEDOUT:
pass
print ">>> Consuming RECEIVED msg of length {}".format(
len(rcv_message))
status = self.lens.consumeOtherSide(rcv_message)
print ">>> STATUS: {}".format(status)
if status == "END":
self.new_iteration(reset=1)
status = self.lens.consumeOtherSide(rcv_message)
print ">>> STATUS: {}".format(status)
if status == "NO TRANSITION":
print "Consuming empty message..."
status = self.lens.consumeOtherSide("")
self.log_trace()
status, msg, transition = self.lens.transitionSelf()
print ">>> STATUS: {}, TRANSITION: {}".format(
status, transition)
if msg is not None:
try:
connection.send(str(msg))
print ">>> SENDING msg of length {}".format(
len(msg))
if len(msg) < 5000:
deco = '#' * 80
print "{}\n{}\n{}".format(deco, msg, deco)
except socket.error as e:
print "socket: {} in send operation".format(e)
connection.close()
break
elif self.lens.role == "UAS":
connection = Server(self.host, self.port, self.timeout, self.bsize)
while True:
if self.cuckoo_session:
self._run_sample()
connection.accept()
connection.settimeout(self.timeout)
self.new_iteration()
rcv_message = ""
while self.status == FUZZ_STATUS_OK:
try:
print ">>> RECEIVING message... "
rcv_message = connection.recv()
if len(rcv_message) == 0:
e = SocketError("Received empty message")
e.errno = 0
raise e
deco = '#' * 80
quoted_rcv_msg = urllib.quote(rcv_message)
print ">>> RECEIVED message:\n{}\n{}\n{}".format(
deco, quoted_rcv_msg, deco)
except Exception as e:
print "socket: {}".format(e)
self._termination_check()
if self.status is FUZZ_STATUS_TERMINATED:
connection.close()
self._crash_check()
self.log_trace()
break
else:
if e.errno == errno.ECONNRESET:
connection.close()
connection.accept()
continue
if e.errno == errno.ETIMEDOUT:
pass
#if rcv_message != "":
print ">>> Consuming RECEIVED msg of length {}".format(
len(rcv_message))
status = self.lens.consumeOtherSide(rcv_message)
print ">>> STATUS: {}".format(status)
if status == "END":
self.new_iteration(reset=1)
status = self.lens.consumeOtherSide(rcv_message)
print ">>> STATUS: {}".format(status)
if status == "NO TRANSITION":
print "Consuming empty message..."
status = self.lens.consumeOtherSide("")
self.log_trace()
status, msg, transition = self.lens.transitionSelf()
print ">>> STATUS: {}, TRANSITION: {}".format(
status, transition)
if msg is not None:
try:
connection.send(str(msg))
print ">>> SENDING msg of length {}".format(
len(msg))
if len(msg) < 5000:
deco = '#' * 80
print "{}\n{}\n{}".format(deco, msg, deco)
except socket.error as e:
print "socket: {} in send operation".format(e)
connection.close()
break
class NetlinkMixin(object):
'''
Generic netlink socket
'''
def __init__(self, family=NETLINK_GENERIC, port=None, pid=None):
super(NetlinkMixin, self).__init__(AF_NETLINK, SOCK_DGRAM, family)
global sockets
self.recv_plugin = self.recv_plugin_init
# 8<-----------------------------------------
# PID init is here only for compatibility,
# later it will be completely moved to bind()
self.addr_pool = AddrPool(minaddr=0xff)
self.epid = None
self.port = 0
self.fixed = True
self.backlog = {0: []}
self.monitor = False
self.callbacks = [] # [(predicate, callback, args), ...]
self.clean_cbs = {} # {msg_seq: [callback, ...], ...}
self.pthread = None
self.backlog_lock = threading.Lock()
self.read_lock = threading.Lock()
self.change_master = threading.Event()
self.lock = LockFactory()
self.buffer_queue = Queue()
self.qsize = 0
self.log = []
self.get_timeout = 3
self.get_timeout_exception = None
if pid is None:
self.pid = os.getpid() & 0x3fffff
self.port = port
self.fixed = self.port is not None
elif pid == 0:
self.pid = os.getpid()
else:
self.pid = pid
# 8<-----------------------------------------
self.groups = 0
self.marshal = Marshal()
# 8<-----------------------------------------
# Set default sockopts
self.setsockopt(SOL_SOCKET, SO_SNDBUF, 32768)
self.setsockopt(SOL_SOCKET, SO_RCVBUF, 1024 * 1024)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.close()
def release(self):
logging.warning("The `release()` call is deprecated")
logging.warning("Use `close()` instead")
self.close()
def register_callback(self, callback,
predicate=lambda x: True, args=None):
'''
Register a callback to run on a message arrival.
Callback is the function that will be called with the
message as the first argument. Predicate is the optional
callable object, that returns True or False. Upon True,
the callback will be called. Upon False it will not.
Args is a list or tuple of arguments.
Simplest example, assume ipr is the IPRoute() instance::
# create a simplest callback that will print messages
def cb(msg):
print(msg)
# register callback for any message:
ipr.register_callback(cb)
More complex example, with filtering::
# Set object's attribute after the message key
def cb(msg, obj):
obj.some_attr = msg["some key"]
# Register the callback only for the loopback device, index 1:
ipr.register_callback(cb,
lambda x: x.get('index', None) == 1,
(self, ))
Please note: you do **not** need to register the default 0 queue
to invoke callbacks on broadcast messages. Callbacks are
iterated **before** messages get enqueued.
'''
if args is None:
args = []
self.callbacks.append((predicate, callback, args))
def unregister_callback(self, callback):
'''
Remove the first reference to the function from the callback
register
'''
cb = tuple(self.callbacks)
for cr in cb:
if cr[1] == callback:
self.callbacks.pop(cb.index(cr))
return
def register_policy(self, policy, msg_class=None):
'''
Register netlink encoding/decoding policy. Can
be specified in two ways:
`nlsocket.register_policy(MSG_ID, msg_class)`
to register one particular rule, or
`nlsocket.register_policy({MSG_ID1: msg_class})`
to register several rules at once.
E.g.::
policy = {RTM_NEWLINK: ifinfmsg,
RTM_DELLINK: ifinfmsg,
RTM_NEWADDR: ifaddrmsg,
RTM_DELADDR: ifaddrmsg}
nlsocket.register_policy(policy)
One can call `register_policy()` as many times,
as one want to -- it will just extend the current
policy scheme, not replace it.
'''
if isinstance(policy, int) and msg_class is not None:
policy = {policy: msg_class}
assert isinstance(policy, dict)
for key in policy:
self.marshal.msg_map[key] = policy[key]
return self.marshal.msg_map
def unregister_policy(self, policy):
'''
Unregister policy. Policy can be:
* int -- then it will just remove one policy
* list or tuple of ints -- remove all given
* dict -- remove policies by keys from dict
In the last case the routine will ignore dict values,
it is implemented so just to make it compatible with
`get_policy_map()` return value.
'''
if isinstance(policy, int):
policy = [policy]
elif isinstance(policy, dict):
policy = list(policy)
assert isinstance(policy, (tuple, list, set))
for key in policy:
del self.marshal.msg_map[key]
return self.marshal.msg_map
def get_policy_map(self, policy=None):
'''
Return policy for a given message type or for all
message types. Policy parameter can be either int,
or a list of ints. Always return dictionary.
'''
if policy is None:
return self.marshal.msg_map
if isinstance(policy, int):
policy = [policy]
assert isinstance(policy, (list, tuple, set))
ret = {}
for key in policy:
ret[key] = self.marshal.msg_map[key]
return ret
def bind(self, groups=0, pid=None, async=False):
'''
Bind the socket to given multicast groups, using
given pid.
* If pid is None, use automatic port allocation
* If pid == 0, use process' pid
* If pid == <int>, use the value instead of pid
'''
if pid is not None:
self.port = 0
self.fixed = True
self.pid = pid or os.getpid()
self.groups = groups
# if we have pre-defined port, use it strictly
if self.fixed:
self.epid = self.pid + (self.port << 22)
super(NetlinkMixin, self).bind((self.epid, self.groups))
else:
# if we have no pre-defined port, scan all the
# range till the first available port
for i in range(1024):
try:
self.port = sockets.alloc()
self.epid = self.pid + (self.port << 22)
super(NetlinkMixin, self).bind((self.epid, self.groups))
# if we're here, bind() done successfully, just exit
break
except SocketError as e:
# pass occupied sockets, raise other exceptions
if e.errno != 98:
raise
else:
# raise "address in use" -- to be compatible
raise SocketError(98, 'Address already in use')
# all is OK till now, so start async recv, if we need
if async:
self._stop = False
self.recv_plugin = self.recv_plugin_queue
self.pthread = threading.Thread(target=self.async_recv)
self.pthread.setDaemon(True)
self.pthread.start()
def getpeername(self):
if self.test.peerNameSucceed:
return ("4.3.2.1", 4321)
else:
raise SocketError(ENOTCONN, "Transport endpoint not connected")
def _connect(self,
url,
params,
ok_codes,
rtype,
description,
redirect=False):
"""Handle fetching, logging, errors and history."""
if params is None and rtype in ('post', 'put'):
# enable empty put/post
params = []
t_start = time.time()
try:
response = self._browser.fetch(url,
params,
ok_codes=ok_codes,
key_file=self._keyfile_path,
cert_file=self._certfile_path,
method=rtype)
except:
etype, value, tback = sys.exc_info()
t_stop = time.time()
t_delta = t_stop - t_start
self.total_time += t_delta
self.step_success = False
self.test_status = 'Failure'
self.logd(' Failed in %.3fs' % t_delta)
if etype is HTTPError:
self._log_response(value.response,
rtype,
description,
t_start,
t_stop,
log_body=True)
if self._dumping:
self._dump_content(value.response)
raise self.failureException, str(value.response)
else:
self._log_response_error(url, rtype, description, t_start,
t_stop)
if etype is SocketError:
raise SocketError("Can't load %s." % url)
raise
t_stop = time.time()
# Log response
t_delta = t_stop - t_start
self.total_time += t_delta
if redirect:
self.total_redirects += 1
elif rtype != 'link':
self.total_pages += 1
else:
self.total_links += 1
if rtype in ('put', 'post', 'get', 'delete'):
# this is a valid referer for the next request
self.setHeader('Referer', url)
self._browser.history.append((rtype, url))
self.logd(' Done in %.3fs' % t_delta)
self._log_response(response, rtype, description, t_start, t_stop)
if self._dumping:
self._dump_content(response)
return response
def sendall(self, data, flags=None):
log.info("Failing send")
raise SocketError()
class TestConfig(unittest.TestCase):
fsock = None
mockdate = "2017-09-06 00:44:47.7"
def _redirectOut(self): # pragma: no cover
self.fsock = open('out.log', 'a+')
sys.stdout = self.fsock
def tearDown(self): # pragma: no cover
if self.fsock is not None:
self.fsock.close()
self.fsock = None
sys.stdout = sys.__stdout__
def setUp(self):
self.config = Config()
self.configXMLString = XMLString
self.config.parseFromString(self.configXMLString)
self.configFBXML = FBXML
def test_timestamp(self):
"""
test getlogtime
:return:
"""
stamp = helpers.getLoggingTime()
self.assertRegexpMatches(stamp, "\d{4}-\d{2}-\d{2}", "should have time stamp,\n" + stamp)
def test_createdir(self):
"""
test dir with file
:return:
"""
helpers.createResultsDir("none")
self.assertEqual(os.path.isdir(helpers.RESULTS_DIR), True, "missing directory " + helpers.RESULTS_DIR)
@patch('os.path.dirname', new=Mock(side_effect=OSError("fail to mock a write", errno.EACCES)))
def test_createdir_raise(self):
"""
test dir raise error
:return:
"""
self.assertRaisesRegexp(OSError, "fail to mock", helpers.createResultsDir, "none")
@patch('os.path.dirname', new=Mock(side_effect=OSError("fail to mock a write", errno.EEXIST)))
def test_createdir_fail(self):
"""
test dir with failing oserror
:return:
"""
self.assertRaisesRegexp(OSError, "fail to mock", helpers.createResultsDir, "none")
def test_dump_none(self):
"""
test none page to a file
:return:
"""
self.assertRaisesRegexp(TypeError, "None", helpers.dumpErrorPage, None)
def test_dump(self):
"""
test dump page to a file
:return:
"""
filename = helpers.dumpErrorPage(self.mockdate)
output = ""
with open("result/" + filename, "r") as fd:
output += fd.readline()
self.assertRegexpMatches(output, "\d{4}-\d{2}-\d{2}", "should have time stamp,\n" + output)
def test_errorontext(self):
"""
test exception from helper's errorOnText
:return:
"""
err = 'Authentication has not been passed: Invalid password'
# not found so no assertion
output = helpers.errorOnText("", 'That password is incorrect.', err)
# should raise if it sees an assertion
self.assertRaisesRegexp(helpers.BingAccountError, "Invalid", helpers.errorOnText, 'That password is incorrect.',
'That password is incorrect.', err)
def test_node(self):
"""
test node's children
:return:
"""
import xml.etree.ElementTree as ET
root = ET.fromstring(self.configXMLString)
node = helpers.getXmlChildNodes(root)
self.assertIsNotNone(node, "should not be null " + str(node))
@patch('sys.version_info')
def test_node_fail(self, mockver):
sys.version_info = [2, 1]
import xml.etree.ElementTree as ET
root = ET.fromstring(self.configXMLString)
node = helpers.getXmlChildNodes(root)
self.assertIsNotNone(node, "should not be null " + str(node))
def test_accounts(self):
self.assertIsNotNone(self.config.accounts)
self.assertEqual(len(self.config.accounts), 1)
accounts = dict()
acc = Config.Account()
acc.accountLogin = "******"
acc.password = "******"
acc.accountType = "Live"
acc.disabled = False
acc.ua_desktop = "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136"
acc.ua_mobile = "mozilla/5.0 (iphone; cpu iphone os 7_0_2 like mac os x) applewebkit/537.51.1 (khtml, like gecko) version/7.0 mobile/11a501 safari/9537.53"
accounts[acc.getRef()] = acc
self.assertEqual(accounts, self.config.accounts)
def test_history(self):
"""
test history parsing
:return:
"""
self.assertRaisesRegexp(TypeError, "None", bingHistory.parse, None)
output = bingHistory.parse("")
self.assertIsNotNone(output, "missing output " + str(output))
page = '<span class="query_t">'
page += '<div id="results_area"></div><div id="sidebar"></div>'
output = bingHistory.parse(page)
self.assertIsNotNone(output, "missing output " + str(output))
page = '<span class="sh_item_qu_query">'
page += '<ul class="sh_dayul"></ul>'
page += ' value == 0'
page += '</span>'
output = bingHistory.parse(page)
self.assertIsNotNone(output, "missing output " + str(output))
page = '<ul class="sh_dayul"> </ul>'
output = bingHistory.parse(page)
self.assertIsNotNone(output, "missing output " + str(output))
output = bingHistory.getBingHistoryTodayURL()
self.assertRegexpMatches(output, "https", "missing url " + str(output))
@patch('helpers.getResponseBody', return_value='"WindowsLiveId":"" "WindowsLiveId":""')
@patch('time.sleep', return_value='')
def test_auth_url(self, timemock, helpmock): # pragma: no cover
"""
test authentication decoding error
:return:
"""
self.assertRaisesRegexp(ValueError, "unknown url type", run, self.config)
@patch('bingAuth.BingAuth.authenticate',
new=Mock(side_effect=SocketError(errno.ECONNREFUSED, "errno.ECONNREFUSED")))
def test_auth_exceptionSock(self):
self.assertRaisesRegexp(SocketError, "", run, self.config)
@patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=SocketError(errno.ECONNRESET, "errno.ECONNRESET")))
def test_auth_exceptionSockReset(self):
self.assertIsNone(run(self.config), "should not return anything")
@patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=helpers.BingAccountError(None)))
def test_auth_exceptionBing(self):
self.assertIsNone(run(self.config), "should not return anything")
@patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=urllib2.URLError("")))
def test_auth_exceptionURL(self):
self.assertIsNone(run(self.config), "should not return anything")
@patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=HTMLParser.HTMLParseError("error")))
def test_auth_exceptionParser(self):
self.assertIsNone(run(self.config), "should not return anything")
@patch('bingAuth.BingAuth.authenticate', new=Mock(side_effect=urllib2.HTTPError("", "", "", "", open("tmp", "a+"))))
def test_auth_exceptionHTTP(self):
self.assertIsNone(run(self.config), "should not return anything")
def test_stringify(self):
self.assertRaisesRegexp(ValueError, "too small", stringify, None, -1)
@patch('urllib2.Request', return_value="")
@patch('helpers.getResponseBody', return_value="")
@patch('urllib2.Request.add_header', return_value=urllib2.Request(bingCommon.BING_URL, bingCommon.HEADERS))
def test_auth_url(self, headermock, helpmock, urlmock):
"""
test auth class
:param headermock:
:param helpmock:
:param urlmock:
:return:
"""
self.assertRaisesRegexp(TypeError, "opener is not", bingAuth.BingAuth, bingCommon.HEADERS, None)
auth = bingAuth.BingAuth(bingCommon.HEADERS, urllib2.OpenerDirector())
self.assertIsNotNone(auth, "should return class")
def test_config(self):
"""
test config module
:return:
"""
configobj = Config()
self.assertIsNotNone(configobj, "should return class")
self.assertIsNotNone(Config.General(), "should return class")
self.assertIsNotNone(ConfigError("ok"), "should return exception")
self.assertIsNotNone(Config.Proxy(), "should return class")
self.assertIsNotNone(Config.EventAccount(), "should return class")
self.assertIsNotNone(Config.Event.Notify(), "should return class")
ifs = Config.Event.IfStatement()
ifs.op = lambda x, y: x
ifs.lhs = lambda x: x
ifs.rhs = "b"
self.assertIsNotNone(str(ifs), "should return class")
self.assertRaisesRegexp(ValueError, "None", ifs.evaluate, None)
self.assertRaisesRegexp(TypeError, "is not of", ifs.evaluate, [])
self.assertIsNotNone(ifs.evaluate(BingRewardsReportItem()))
spec = Config.Event.Specifier()
self.assertIsNotNone(spec, "should return class")
self.assertRaisesRegexp(ValueError, "is None", spec.evaluate, None, BingRewardsReportItem())
self.assertRaisesRegexp(TypeError, "list", spec.evaluate, [], BingRewardsReportItem())
self.assertRaisesRegexp(ValueError, "is None", spec.evaluate, [], None)
self.assertRaisesRegexp(TypeError, "not of BingRewardsReportItem type", spec.evaluate, [], self.config)
self.assertIsNotNone(spec.evaluate("%a", BingRewardsReportItem()), "should return string")
dist = os.path.join(os.path.dirname(__file__), "..", "config.xml")
self.assertIsNone(configobj.parseFromFile(dist), "should be none")
self.assertRaisesRegexp(ValueError, "_configFile_ is None", configobj.parseFromFile, None)
self.assertRaisesRegexp(ValueError, "is None", self.config.parseFromString, None)
self.assertRaisesRegexp(ConfigError, "Invalid subnode", configobj.parseFromString, InvalidXML)
self.assertRaisesRegexp(ConfigError, "is not found", configobj.parseFromString, LOGINXML)
self.assertRaisesRegexp(ConfigError, "is not found", configobj.parseFromString, PWDXML)
self.assertRaisesRegexp(ConfigError, "should be either set", self.config.parseFromString, PROXYLOGINXML)
self.assertRaisesRegexp(KeyError, "_specifier_ is not", validateSpecifier, "%not")
self.assertRaisesRegexp(ConfigError, "Invalid subnode", self.config.parseFromString, FBXML)
def test_config_attr(self):
self.assertRaisesRegexp(ConfigError, "MUST", self.config.parseFromString, FLOAT)
self.assertRaisesRegexp(ConfigError, "MUST", self.config.parseFromString, INT)
self.assertRaisesRegexp(ConfigError, "must", self.config.parseFromString, NONFLOAT)
self.assertRaisesRegexp(ConfigError, "must", self.config.parseFromString, NONINT)
def test_config_notify(self):
self.assertRaisesRegexp(ConfigError, "is not found", self.config.parseFromString, NONREF)
self.assertRaisesRegexp(ConfigError, "is not found", self.config.parseFromString, NONACCREF)
self.assertRaisesRegexp(ConfigError, "not supported", self.config.parseFromString, NONEV)
def test_config_retry(self):
self.assertRaisesRegexp(ConfigError, "is not found", self.config.parseFromString, RETRY)
self.assertRaisesRegexp(ConfigError, "must be", self.config.parseFromString, INVRETRY)
self.assertRaisesRegexp(ConfigError, "MUST BE", self.config.parseFromString, NEGRETRY)
self.assertRaisesRegexp(ConfigError, "is not found", self.config.parseFromString, RETRYCNT)
self.assertRaisesRegexp(ConfigError, "must be", self.config.parseFromString, INVRETRYCNT)
self.assertRaisesRegexp(ConfigError, "MUST BE", self.config.parseFromString, NEGRETRYCNT)
def test_config_if(self):
self.assertRaisesRegexp(ConfigError, "is invalid", self.config.parseFromString, NONIF2)
self.assertRaisesRegexp(ConfigError, "is invalid", self.config.parseFromString, NONIFRHS)
self.assertRaisesRegexp(ConfigError, "is invalid", self.config.parseFromString, NONIFOP)
def test_event(self):
"""
test event
:return:
"""
self.assertIsNone(EventsProcessor.onScriptFailure(self.config, Exception()), "should be none")
self.assertIsNone(EventsProcessor.onScriptComplete(self.config), "should be none")
self.assertRaisesRegexp(ConfigError, "not found", self.config.parseFromString, EVENT)
self.config.parseFromString(EVENTLESS)
self.assertRaisesRegexp(Exception, ".*", EventsProcessor.onScriptFailure, self.config, Exception())
self.assertIsNone(EventsProcessor.onScriptComplete(self.config), "should be none")
ep = EventsProcessor(self.config, BingRewardsReportItem())
self.assertIsNotNone(ep.processReportItem(), "should not be none and be done")
@patch('main.earnRewards', return_value=None)
@patch('eventsProcessor.EventsProcessor.processReportItem', return_value=(-1, None))
def test_event_dontcare(self, mockep, mockmain):
# not retry nor ok with -1
self.assertIsNone(processAccount(self.config), "should return nothing")
def test_event_getEvent_returnsEvent(self):
"""
test onScriptFailure using echo from xml config string
:return:
"""
event = self.config.getEvent(Config.Event.onScriptFailure)
self.assertIsNotNone(event)
self.assertTrue(len(event.notifies) == 1)
self.assertEqual(event.notifies[0].cmd, "echo")
def test_event_getEvent_returnsNoneIfEventDoesntExist(self):
"""
test no event call does not exist
:return:
"""
self.assertIsNone(self.config.getEvent("does_not_exist"))
self.assertRaisesRegexp(ValueError, "None", self.config.getEvent, None)
def test_reward_bfp_hit(self):
self._rewards_hit(bfp.RewardV1())
self._rewards_hit(bdp.Reward())
@patch('helpers.getResponseBody')
def _rewards_hit(self, classobj, helpmock):
"""
test rewards object
:return:
"""
self.config.proxy = False
reward = BingRewards(bingCommon.HEADERS, "", self.config)
page = '"WindowsLiveId":"" "WindowsLiveId":"" '
page += 'action="0" value="0" '
page += 'value= "0" NAP value="0" '
page += 'ANON value="0" '
page += 'id="t" value="0" '
page += '<div> 999 livetime points</div> '
helpmock.return_value = page
# if not login should have not found error for url
self.assertIsNotNone(reward.getLifetimeCredits, "Should return int")
page = "t.innerHTML='100'"
helpmock.return_value = page
self.assertIsNotNone(reward.getRewardsPoints(), "should not be None")
self.assertRaisesRegexp(TypeError, "not an instance", reward.process, None, True)
# NONE case
newbfp = classobj
newbfp.tp = None
rewards = [newbfp]
self.assertIsNotNone(reward.process(rewards, True), "handle not none")
# HIT case
newbfp.tp = mock.Mock()
newbfp.tp = [0, 1, 2, 3, bfp.RewardV1.Type.Action.HIT]
# SEARCH case
newbfp.tp = mock.Mock()
newbfp.tp = [0, 1, 2, 3, bfp.RewardV1.Type.Action.SEARCH]
newbfp.progressCurrent = 100
rewards = [newbfp]
self.assertIsNotNone(reward.process(rewards, True), "should return res")
self.assertRaisesRegexp(TypeError, "not an instance", reward.printResults, None, True)
result = mock.Mock()
result.action = bfp.RewardV1.Type.Action.SEARCH
result.isError = True
result.o = newbfp
result.message = "done"
newbfp.progressCurrent = 1
newbfp.progressMax = 100
newbfp.url = "http:0.0.0.0"
self.assertIsNone(reward.printResults([result], True), "should return None")
self.assertRaisesRegexp(TypeError, "rewards is not", reward.printRewards, None)
rewards[0].isDone = True
self.assertIsNone(reward.printRewards(rewards), "should return None")
self.assertRaisesRegexp(TypeError, "reward is not", reward.RewardResult, None)
self.assertIsNotNone(reward.RewardResult(newbfp), "should return class")
proxy = mock.Mock()
proxy.login = True
proxy.password = "******"
proxy.url = "http://127.0.0.1"
proxy.protocols = "http"
self.config.proxy = proxy
self.assertIsNotNone(BingRewards(bingCommon.HEADERS, "", self.config), "should return class")
proxy.login = False
self.config.proxy = proxy
self.assertIsNotNone(BingRewards(bingCommon.HEADERS, "", self.config), "should return class")
self.assertRaisesRegexp(ConfigError, "not found", self.config.parseFromString, PROTXML)
self.assertRaisesRegexp(ConfigError, "not found", self.config.parseFromString, URLXML)
def _setup_https_tunnel(self):
sock = self.sock
host = self._tunnel_host
port = self._tunnel_port
try:
lines = []
lines.append('CONNECT %s:%d HTTP/1.1' % (host, port))
lines.append('Host: %s:%d' % (host, port))
if self._tunnel_headers:
for item in self._tunnel_headers.items():
lines.append('%s: %s' % item)
data = '\r\n'.join(lines) + '\r\n\r\n'
sock.sendall(data.encode())
data = b''
code = 0
pos = -1
while True:
s = sock.recv(4096)
if not s:
if code == 0:
raise SocketError("Tunnel connection failed: %r" %
data)
break
data += s
if code == 0 and b'\r\n' in data:
version, code, message = data.split(b' ', 2)
if code != b'200':
sock.close()
raise SocketError("Tunnel connection failed: %s %s" %
(code, message.strip()))
pos = data.find(b'\r\n\r\n')
if pos > 0:
break
tls_conn = tlslite.TLSConnection(sock)
try:
tls_conn.handshakeClientCert(serverName=host)
except Exception:
sock.close()
raise
try:
ssl.match_hostname(tlslite_getpeercert(tls_conn), host)
except Exception:
tls_conn.close()
raise
except SocketTimeout as e:
raise ConnectTimeoutError(
self, "Connection to %s timed out. (connect timeout=%s)" %
(self.host, self.timeout))
except SocketError as e:
raise NewConnectionError(
self, "Failed to establish a new connection: %s" % e)
# patch fileno,
# let urllib3.util.connection.is_connection_dropped work as expected
# tls_conn.fileno = partial(self._origin_sock.fileno) # here we always got fileno = -1
tls_conn.fileno = partial(sock.fileno)
# patch getpeercert
tls_conn.getpeercert = partial(tlslite_getpeercert, tls_conn)
self.sock = tls_conn
def sendall(self, data, flags=None):
raise SocketError()
def raise_socket_error(*args, **kwargs):
raise SocketError()
def send(self, data):
while True:
try:
return self.connection.send(data)
except OpenSSL.SSL.SysCallError as e:
raise SocketError(str(e))
def _proxy_function(self, func_name, *args, **kw):
result_error_set_list = []
protocol_error_list = []
socket_error_list = []
fault_error_list = []
count = 0
serverproxy_list = self._serverproxy_list
# we have list of tuple (uri, ServerProxy()). Sort by uri having oldest failure
serverproxy_list.sort(
key=lambda x: global_server_proxy_uri_failure_time.get(x[0], 0))
while True:
retry_server_list = []
for uri, server_proxy in serverproxy_list:
func = getattr(server_proxy, func_name)
failure = True
try:
# Cloudooo return result in (200 or 402, dict(), '') format or just based type
# 402 for error and 200 for ok
result_set = func(*args, **kw)
except SocketError as e:
message = 'Socket Error: %s' % (repr(e) or 'undefined.')
socket_error_list.append(message)
retry_server_list.append((uri, server_proxy))
except ProtocolError as e:
# Network issue
message = "%s: %s %s" % (e.url, e.errcode, e.errmsg)
if e.errcode == -1:
message = "%s: Connection refused" % (e.url)
protocol_error_list.append(message)
retry_server_list.append((uri, server_proxy))
except Fault as e:
# Return not supported data types
fault_error_list.append(e)
else:
failure = False
if not (failure):
try:
response_code, _, _ = result_set
except ValueError:
# Compatibility for old oood, result is based type, like string
response_code = 200
if response_code == 200:
return result_set
else:
# If error, try next one
result_error_set_list.append(result_set)
# Still there ? this means we had no result,
# avoid using same server again
global_server_proxy_uri_failure_time[uri] = int(DateTime())
# All servers are failed
if count == self._ooo_server_retry or len(retry_server_list) == 0:
break
count += 1
serverproxy_list = retry_server_list
# Check error type
# Return only one error result for compability
if len(result_error_set_list):
return result_error_set_list[0]
if len(protocol_error_list):
raise ConversionError(
"Protocol error while contacting OOo conversion: "
"%s" % (','.join(protocol_error_list)))
if len(socket_error_list):
raise SocketError("%s" % (','.join(socket_error_list)))
if len(fault_error_list):
raise fault_error_list[0]
def fail(message):
if 'resolved' not in state:
state['resolved'] = True
self._cb = None
router_error(SocketError(message))
# Copy-pasted from Python 3.5 source code
def _decref_socketios(self):
if self._makefile_refs > 0:
self._makefile_refs -= 1
if self._closed:
self.close()
def recv(self, *args, **kwargs):
try:
data = self.connection.recv(*args, **kwargs)
except OpenSSL.SSL.SysCallError as e:
<<<<<<< HEAD
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
return b""
else:
raise SocketError(str(e))
except OpenSSL.SSL.ZeroReturnError:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return b""
=======
if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
return b''
else:
raise SocketError(str(e))
except OpenSSL.SSL.ZeroReturnError as e:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return b''
>>>>>>> 71358189c5e72ee2ac9883b408a2f540a7f5745e
else:
raise
except OpenSSL.SSL.WantReadError:
def run(self, text, options, path):
"""
Sends the TTS request to ImTranslator, captures the audio from
the returned SWF, and transcodes to MP3.
Because ImTranslator sometimes raises various errors, both steps
of this (i.e. downloading the page and dumping the audio) may be
retried up to three times.
"""
output_wavs = []
output_mp3s = []
require = dict(size_in=4096)
logger = self._logger
try:
for subtext in self.util_split(text, 400):
for i in range(1, 4):
try:
logger.info("ImTranslator net_stream: attempt %d", i)
result = self.net_stream(
('http://imtranslator.net/translate-and-speak/'
'sockets/tts.asp',
dict(text=subtext, vc=options['voice'],
speed=options['speed'], FA=1)),
require=dict(mime='text/html', size=256),
method='POST',
)
result = self._RE_SWF.search(result)
if not result or not result.group():
raise EnvironmentError('500b', "cannot find SWF"
"path in payload")
result = result.group()
except (EnvironmentError, IOError) as error:
if getattr(error, 'code', None) == 500:
logger.warn("ImTranslator net_stream: got 500")
elif getattr(error, 'errno', None) == '500b':
logger.warn("ImTranslator net_stream: no SWF path")
elif 'timed out' in format(error):
logger.warn("ImTranslator net_stream: timeout")
else:
logger.error("ImTranslator net_stream: %s", error)
raise
else:
logger.info("ImTranslator net_stream: success")
break
else:
logger.error("ImTranslator net_stream: exhausted")
raise SocketError("unable to fetch page from ImTranslator "
"even after multiple attempts")
output_wav = self.path_temp('wav')
output_wavs.append(output_wav)
for i in range(1, 4):
try:
logger.info("ImTranslator net_dump: attempt %d", i)
self.net_dump(output_wav, result)
except RuntimeError:
logger.warn("ImTranslator net_dump: failure")
else:
logger.info("ImTranslator net_dump: success")
break
else:
logger.error("ImTranslator net_dump: exhausted")
raise SocketError("unable to dump audio from ImTranslator "
"even after multiple attempts")
if len(output_wavs) > 1:
for output_wav in output_wavs:
output_mp3 = self.path_temp('mp3')
output_mp3s.append(output_mp3)
self.cli_transcode(output_wav, output_mp3, require=require)
self.util_merge(output_mp3s, path)
else:
self.cli_transcode(output_wavs[0], path, require=require)
finally:
self.path_unlink(output_wavs, output_mp3s)
def test_open_connection_error(self, mock_connection, mock_path):
# raise exception
mock_connection.connect.side_effect = SocketError()
with self.assertRaises(CommunicationError):
self._socket.open()