def __init__(self, auth_url, username, api_key, pool=None, proxy=None,
extra_headers=None, verbose=False):
self.auth_url = auth_url
self.username = username
self.api_key = api_key
self.storage_url = None
self.auth_token = None
self.pool = pool
if proxy:
if ":" in proxy:
addr, port = proxy.rsplit(":", 1)
port = int(port)
else:
addr, port = proxy, 8000
endpoint = TCP4ClientEndpoint(reactor, addr, port)
self.agent = ProxyAgent(endpoint, pool=self.pool)
else:
contextFactory = WebClientContextFactory()
contextFactory.noisy = False
self.agent = Agent(reactor, contextFactory, pool=self.pool)
self.extra_headers = extra_headers
self.verbose = verbose
def __init__(self, auth_url, username, api_key, pool=None,
extra_headers=None, verbose=False):
self.auth_url = auth_url
self.username = username
self.api_key = api_key
self.storage_url = None
self.auth_token = None
contextFactory = WebClientContextFactory()
contextFactory.noisy = False
self.pool = pool
self.agent = Agent(reactor, contextFactory, pool=self.pool)
self.extra_headers = extra_headers
self.verbose = verbose
def _make_request(self, method, url, headers=None, data=None, files=None,
timeout=None, context_factory=None,
data_limit=None):
context_factory = (context_factory if context_factory is not None
else WebClientContextFactory())
if headers is not None:
headers = dict((k.encode("utf-8"), [x.encode("utf-8") for x in v])
for k, v in headers.items())
if data is not None:
data = data.encode("utf-8")
if files is not None:
files = dict([
(key,
(value['file_name'],
value['content_type'],
StringIO(base64.b64decode(value['data']))))
for key, value in files.iteritems()])
agent = self.agent_class(reactor, contextFactory=context_factory)
http_client = self.http_client_class(agent)
d = http_client.request(method, url, headers=headers, data=data,
files=files, timeout=timeout)
d.addCallback(self._ensure_data_limit, data_limit)
return d
def test_init(self):
url = 'https://example.org/abcdef'
version = '2.0'
proxy = Proxy(url, version, contextFactory=WebClientContextFactory())
self.assertEquals(proxy.url, url)
self.assertEquals(proxy.version, version)
def __init__(self,
auth_url,
username,
api_key,
pool=None,
extra_headers=None,
verbose=False):
self.auth_url = auth_url
self.username = username
self.api_key = api_key
self.storage_url = None
self.auth_token = None
contextFactory = WebClientContextFactory()
contextFactory.noisy = False
self.pool = pool
self.agent = Agent(reactor, contextFactory, pool=self.pool)
self.extra_headers = extra_headers
self.verbose = verbose
def __init__(self,
reactor,
contextFactory=WebClientContextFactory(),
connectTimeout=None,
bindAddress=None):
Agent.__init__(self, reactor, contextFactory)
self._connectTimeout = connectTimeout
self._bindAddress = bindAddress
class Repeater(object):
USER_AGENT = 'STXNext Intranet 2 Cron task'
contextFactory = WebClientContextFactory()
client = Agent(reactor, contextFactory)
def __init__(self, *actions):
self.headers = {
'User-Agent': [self.USER_AGENT],
'X-Intranet-Cron': [config['CRON_SECRET_KEY']]
}
self.actions = {}
for action in actions:
self.actions[action.name] = action.url_producer
self.file_path = config['REPEATER_FILE']
def pending(self):
""" returns a list of pending entries """
result = []
yesterday = previous_day(datetime.date.today())
cron_url = config['CRON_URL']
with PickleStore(self.file_path) as store:
for date, action_name in store.get_pending(yesterday,
self.actions.keys()):
result.append(
('%s%s' % (cron_url, self.actions[action_name](date)),
partial(self.update, date, action_name)))
return result
def update(self, date, action_name, done):
with PickleStore(self.file_path) as store:
store.update(date, action_name, done)
def on_success(self, url, callback, resp):
LOG(u"Repeater %s succeeded with status %s" % (url, resp.code))
callback(resp.code == 200)
def on_failure(self, url, callback, err):
EXCEPTION(u"Repeater %s failed %s" % (url, err))
callback(False)
def __call__(self):
DEBUG(u"Repeater starting")
i = 0
try:
for url, callback in self.pending():
DEBUG(u"Will call action %s" % (url, ))
deferred = self.client.request('GET', url,
Headers(self.headers))
deferred.addCallbacks(partial(self.on_success, url, callback),
partial(self.on_failure, url, callback))
i += 1
except:
EXCEPTION(u"Repeater could not start")
DEBUG(u"Repeater started %s jobs" % (i, ))
class URLCronTask(object):
USER_AGENT = 'STXNext Intranet 2 Cron task'
contextFactory = WebClientContextFactory()
client = Agent(reactor, contextFactory)
def __init__(self, task_name, url, repeats=None):
self.busy = False
self.task_name = task_name
self.url = '%s%s' % (config['CRON_URL'], url)
self.repeats = repeats
self.repeated = 0
def get_headers(self):
""" Generate request headers (as a dictionary) """
return {
'User-Agent': [self.USER_AGENT],
'X-Intranet-Cron': [config['CRON_SECRET_KEY']]
}
def request(self, url, headers, method='GET'):
LOG(u'Will request URL %s with headers %s, method %s' %
(url, pformat(headers), method))
deferred = self.client.request(method, url, Headers(headers))
deferred.addCallbacks(self.on_success, self.on_failure)
def on_failure(self, err):
self.busy = False
EXCEPTION(u"Cron task [%s] failed: %s" % (self.task_name, err))
def on_success(self, resp):
self.busy = False
LOG(u'Cron function [%s] finished (status code %s)' %
(self.task_name, resp.code))
def __call__(self):
LOG(u'Cron function [%s] starting (%s)' %
(self.task_name, self.repeated))
if not self.busy:
self.busy = True
self.repeated = 0
self.request(self.url, self.get_headers())
elif (self.repeats is not None) and self.repeated >= self.repeats:
WARN(u"Overriding busy action [%s] on %s/%s time" %
(self.task_name, self.repeated, self.repeats))
self.repeated = 0
self.request(self.url, self.get_headers())
else:
self.repeated += 1
WARN(u'Action [%s] is busy (%s/%s)' %
(self.task_name, self.repeated, self.repeats))
def test_http_request_with_custom_context_factory(self):
self.set_render(lambda r: "Yay")
ctxt = WebClientContextFactory()
class FakeAgent(Agent):
def __init__(slf, reactor, contextFactory=None):
self.assertEqual(contextFactory, ctxt)
super(FakeAgent, slf).__init__(reactor, contextFactory)
request = yield http_request_full(self.url,
'',
context_factory=ctxt,
agent_class=FakeAgent)
self.assertEqual(request.delivered_body, "Yay")
self.assertEqual(request.code, http.OK)
def getContext(self, hostname, port):
ctx = WebClientContextFactory.getContext(self, hostname, port)
if self.cfile:
try:
x509Obj = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, self.cfile)
ctx.use_certificate(x509Obj)
except:
ctx.use_certificate_file(self.cfile)
if self.kfile:
try:
pkey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, self.kfile)
ctx.use_privatekey(pkey)
except:
ctx.use_privatekey_file(self.kfile)
return ctx
def http_request_full(url,
data=None,
headers={},
method='POST',
timeout=None,
data_limit=None,
context_factory=None,
agent_class=None,
reactor=None):
if reactor is None:
# The import replaces the local variable.
from twisted.internet import reactor
if agent_class is None:
agent_class = Agent
context_factory = context_factory or WebClientContextFactory()
agent = agent_class(reactor, contextFactory=context_factory)
d = agent.request(method, url, mkheaders(headers),
StringProducer(data) if data else None)
def handle_response(response):
return SimplishReceiver(response, data_limit).deferred
d.addCallback(handle_response)
if timeout is not None:
cancelling_on_timeout = [False]
def raise_timeout(reason):
if not cancelling_on_timeout[0] or reason.check(HttpTimeoutError):
return reason
return Failure(HttpTimeoutError("Timeout while connecting"))
def cancel_on_timeout():
cancelling_on_timeout[0] = True
d.cancel()
def cancel_timeout(r, delayed_call):
if delayed_call.active():
delayed_call.cancel()
return r
d.addErrback(raise_timeout)
delayed_call = reactor.callLater(timeout, cancel_on_timeout)
d.addCallback(cancel_timeout, delayed_call)
return d
def test_http_request_with_custom_context_factory(self):
self.set_render(lambda r: "Yay")
agents = []
ctxt = WebClientContextFactory()
def stashing_factory(reactor, contextFactory=None, pool=None):
agent = self.fake_http.get_agent(
reactor, contextFactory=contextFactory, pool=pool)
agents.append(agent)
return agent
request = yield http_request_full(
self.url, '', context_factory=ctxt, agent_class=stashing_factory)
self.assertEqual(request.delivered_body, "Yay")
self.assertEqual(request.code, http.OK)
[agent] = agents
self.assertEqual(agent.contextFactory, ctxt)
def test_callRemote(self):
"""
The test itself passes, but trial raises "Reactor was unclean" after
tearDown.. Might be related to
http://twistedmatrix.com/trac/ticket/5118
"""
data = 'some random string'
addr = 'https://localhost:%s' % self.portNumber
proxy = Proxy(addr,
jsonrpc.VERSION_1,
contextFactory=WebClientContextFactory())
d = proxy.callRemote('echo', data)
def finished(result):
self.assertEquals(result, data)
d.addCallback(finished)
return d
def __init__(self,
url,
version=jsonrpc.VERSION_1,
connectTimeout=None,
credentials=Anonymous(),
contextFactory=WebClientContextFactory()):
"""
@type url: str
@param url: URL of the RPC server. Supports HTTP and HTTPS for now,
more might come in the future.
@type version: int
@param version: Which JSON-RPC version to use? The default is 1.0.
@type connectTimeout: float
@param connectTimeout: Connection timeout. Note that we don't connect
when creating this object, but in callRemote, so the timeout
will apply to callRemote.
@type credentials: twisted.cred.credentials.ICredentials
@param credentials: Credentials for basic HTTP authentication.
Supported are Anonymous and UsernamePassword classes.
@type contextFactory: twisted.internet.ssl.ClientContextFactory
@param contextFactory: A context factory for SSL clients.
"""
self.url = url
self.version = version
if not isinstance(credentials, (Anonymous, UsernamePassword)):
raise NotImplementedError("'%s' credentials are not supported" %
type(credentials))
self.agent = Agent(reactor,
connectTimeout=connectTimeout,
contextFactory=contextFactory)
self.credentials = credentials
self.auth_headers = None
def __init__(self,reactor,api_key,api_secret):
# Bitcoin.de API URI
apihost = 'https://api.bitcoin.de'
apiversion = 'v1'
orderuri = apihost + '/' + apiversion + '/' + 'orders'
tradeuri = apihost + '/' + apiversion + '/' + 'trades'
accounturi = apihost + '/' + apiversion + '/' + 'account'
# set initial nonce
self.nonce = int(time.time())
self.reactor = reactor
pool = HTTPConnectionPool(reactor) # Actually reusing the connection leads to correct credits
pool.maxPersistentPerHost = 1
self.contextFactory = WebClientContextFactory()
self.agent = Agent(self.reactor, self.contextFactory,pool=pool)
self.api_key = api_key
self.api_secret = api_secret
self.calls = {}
# Method,uri,required params with allowed values,credits,field to return (after credits/pages are stripped)
# Orders
self.calls['showOrderbook'] = ['GET',orderuri,{'type':['sell','buy']},2]
self.calls['showOrderbookCompact'] = ['GET',orderuri+'/compact',{},3]
self.calls['createOrder'] = ['POST',orderuri,{'type':['sell','buy'],'max_amount':[],'price':[]},1]
self.calls['deleteOrder'] = ['DELETE',orderuri,{'order_id':[]},2]
self.calls['showMyOrders'] = ['GET',orderuri+'/my_own',{},2] # Fix: all arguments are optional
self.calls['showMyOrderDetails'] = ['GET',orderuri,{'order_id':[]},2]
# Trades
self.calls['executeTrade'] = ['POST',tradeuri,{'order_id':[],'amount':[]},1]
self.calls['showMyTradeDetails'] = ['GET',tradeuri,{'trade_id':[]},3]
self.calls['showMyTrades'] = ['GET',tradeuri,{},3]
self.calls['showPublicTradeHistory'] = ['GET',tradeuri+'/history',{'since_tid':[]},3]
# Account
self.calls['showAccountInfo'] = ['GET',accounturi,{},2]
self.calls['showAccountLedger'] = ['GET',accounturi+'/ledger',{},3]
# Other
self.calls['showRates'] = ['GET',apihost+'/'+apiversion+'/rates',{},3]
class BaseFetcher(object):
USER_AGENT = 'Intranet Bug Fetcher'
contextFactory = WebClientContextFactory()
client = Agent(reactor, contextFactory)
SLEEP_PERIOD = 0.1
CACHE_TIMEOUT = 3 * 60 # 3 minutes
redirect_support = False
def __init__(self, tracker, credentials, login_mapping):
self.tracker = tracker
self.login = credentials.login
self.password = credentials.password
self.user = credentials.user
self.login_mapping = login_mapping
self.bugs = {}
self.done = False
self.error = None
self.cache_key = None
self.dependson_and_blocked_status = {}
def run(self):
""" start fetching tickets """
try:
# start asynchronous ticket fetching
self.fetch()
except:
# failed to start ticket fetching
self.done = True
raise
def request(self, url, headers, on_success, on_failure=None, method='GET', body=None):
LOG(u'Will request URL %s' % (url, ))
if on_failure is None:
on_failure = self.failed
deferred = self.client.request(
method,
url,
Headers(headers),
None if body is None else StringProducer(body)
)
def redirecting_on_success(resp):
if resp.code == 302:
LOG(u"Redirect (302) found in response")
location = resp.headers.getRawHeaders('location')[0]
self.request(location, headers, on_success, on_failure, 'GET', None)
else:
on_success(resp)
deferred.addCallbacks(redirecting_on_success if self.redirect_support else on_success, on_failure)
def failed(self, err):
self.fail(err)
EXCEPTION(u"Fetcher for tracker %s failed: %s" % (self.tracker.name, err))
def success(self):
self.done = True
def get_headers(self):
""" Generate request headers (as a dictionary) """
return {
'User-Agent': [self.USER_AGENT]
}
def fetch_user_tickets(self):
""" Start fetching tickets for current user """
raise NotImplementedError()
def fetch_all_tickets(self):
""" Start fetching tickets for all users in mapping """
raise NotImplementedError()
def fetch_user_resolved_bugs(self):
""" Start fetching fixable tickets for current user """
raise NotImplementedError()
def fetch_all_resolved_bugs(self):
""" Start fetching fixable tickets for all users """
raise NotImplementedError()
def fetch_bugs_for_query(self, ticket_ids, project_selector, component_selector, version):
""" Start fetching all bugs matching given criteria """
raise NotImplementedError()
def fetch_resolved_bugs_for_query(self, ticket_id, project_selector, component_selector, version):
""" Start fetching resolved bugs matching given criteria """
raise NotImplementedError()
def fetch_bug_titles_and_depends_on(self, ticket_ids):
""" Start fetching bug titles and bug depends_on for bugs with given ids """
# TODO other implementations
self.success()
def fetch_dependons_for_ticket_ids(self, ticket_ids):
""" Start recursively fetching dependons for ticket ids """
raise NotImplementedError()
def fetch_scrum(self, sprint_name, project_id):
raise NotImplementedError()
def isReady(self):
""" Check if this fetcher is done """
return self.done
def resolve_user(self, orig_login):
login = orig_login.lower()
if login in self.login_mapping:
return self.login_mapping[login]
else:
return User(name=orig_login, email=orig_login)
def resolve(self, bug):
bug.owner = self.resolve_user(bug.owner)
bug.reporter = self.resolve_user(bug.reporter)
bug.project_id = SelectorMapping(self.tracker).match(
bug.id, bug.project_name, bug.component_name, bug.version,
)
def __iter__(self):
""" iterate over fetched tickets """
if self.cache_key and self.error is None: # cache bugs if key was designeated and no error occured
memcache.set(self.cache_key, self.bugs, timeout=self.CACHE_TIMEOUT)
DEBUG(u"Cached %s bugs for key %s" % (len(self.bugs), self.cache_key))
for bug in self.bugs.itervalues():
self.resolve(bug)
yield bug
def responded(self, resp, on_success=None):
""" Called when server returns response headers """
if resp.code == 200:
on_success = on_success or self.received
resp.deliverBody(SimpleProtocol(on_success, self.failed))
else:
self.fail(FetchException(u'Received response %s' % (resp.code, )))
def received(self, data):
""" Called when server returns whole response body """
try:
for bug in self.parse(data):
self.bugs[bug.id] = bug
except BaseException, e:
EXCEPTION(u"Could not parse tracker response")
self.fail(e)
else:
def test_init_agent(self):
proxy = Proxy('', '', contextFactory=WebClientContextFactory())
self.assertTrue(isinstance(proxy.agent, Agent))
self.assertTrue(
isinstance(proxy.agent._contextFactory, ssl.ClientContextFactory))
def printValue(value):
print "Result: %s" % str(value)
def printError(error):
print 'error', error.value
def shutDown(data):
print "Shutting down reactor..."
reactor.stop()
address = 'https://localhost:8999'
proxy = Proxy(address, contextFactory=WebClientContextFactory())
ds = []
d = proxy.callRemote('echo', ['ajajaj', 'bjbjbj'])
d.addCallbacks(printValue, printError)
ds.append(d)
d = proxy.callRemote('add', 14, 15)
d.addCallbacks(printValue, printError)
ds.append(d)
d = proxy.callRemote('mysql_first_user')
d.addCallbacks(printValue, printError)
ds.append(d)
d = proxy.callRemote('none')