def test_003():
u = "http://*****:*****@%s:%s/auth" % (HOST, PORT)
r = request(u)
t.eq(r.status_int, 200)
u = "http://*****:*****@%s:%s/auth" % (HOST, PORT)
r = request(u)
t.eq(r.status_int, 403)
def test_003():
u = "http://*****:*****@%s:%s/auth" % (HOST, PORT)
r = request(u)
t.eq(r.status_int, 200)
u = "http://*****:*****@%s:%s/auth" % (HOST, PORT)
r = request(u)
t.eq(r.status_int, 403)
def ping(self, issue, command, created, creator):
"""
note: we might send multiple pings if a command was of more than one class
"""
for row in self.graph.queryd(
"SELECT DISTINCT ?cls WHERE { ?cmd a ?cls }",
initBindings={'cmd': command}):
payload = jsonlib.write({
'issue': issue,
'commandClass': row['cls'],
'command': command,
'created': created,
'creator': creator,
})
for receiver in [
"http://bang:9030/dispatch/newCommand",
# ^ this one should be dispatching to the rest, but i
# don't have a proper PSHB setup yet
"http://bang:9055/newCommand",
"http://bang:9072/newCommand",
]:
try:
restkit.request(
method="POST",
url=receiver,
body=payload,
headers={"Content-Type": "application/json"})
except Exception, e:
log.error(e)
def postIdenticaOauth():
""" not working yet. last tried on 2010-05 """
from restkit import OAuthFilter, request
import restkit.oauth2
consumer = restkit.oauth2.Consumer(key=oauthKey, secret=oauthSecret)
request_token_url = "http://identi.ca/api/oauth/request_token"
auth = OAuthFilter(('*', consumer))
if 1:
# The request.
resp = request(request_token_url, filters=[auth])
print resp.__dict__
print resp.body_string()
else:
tok = restkit.oauth2.Token(oauth_token, oauth_token_secret)
resp = restkit.request(
"http://identi.ca/api/statuses/friends_timeline.json",
filters=[OAuthFilter(('*', consumer, tok))],
method="GET")
print resp.body_string()
print resp
resp = restkit.request("http://identi.ca/api/statuses/update.json",
filters=[OAuthFilter(('*', consumer, tok))],
method="POST",
body=jsonlib.dumps({'status' : 'first oauth update'}))
print resp.body_string()
print resp
def ping(self, issue, command, created, creator):
"""
note: we might send multiple pings if a command was of more than one class
"""
for row in self.graph.queryd(
"SELECT DISTINCT ?cls WHERE { ?cmd a ?cls }",
initBindings={'cmd' : command}):
payload = jsonlib.write({
'issue' : issue,
'commandClass' : row['cls'],
'command' : command,
'created' : created,
'creator' : creator,
})
for receiver in [
"http://bang:9030/dispatch/newCommand",
# ^ this one should be dispatching to the rest, but i
# don't have a proper PSHB setup yet
"http://bang:9055/newCommand",
"http://bang:9072/newCommand",
]:
try:
restkit.request(
method="POST", url=receiver,
body=payload,
headers={"Content-Type" : "application/json"})
except Exception, e:
log.error(e)
def updateLcd(self):
whole = "%-147s%-21s" % (self.message, self.lastLine)
try:
restkit.request(url=self.putUrl,
method="PUT",
body=whole,
headers={"content-type":"text/plain"})
except socket.error, e:
log.warn("update lcd failed, %s" % e)
def check_doc_service(self):
try:
request("{host}:{port}/".format(host=self.doc_service_host,
port=self.doc_service_port))
except RequestError as e:
logger.info('DocService connection error, message {}'.format(e),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_DOC_SERVICE_CONN_ERROR},
{}))
raise e
else:
return True
def test_002():
u = "http://%s:%s" % (HOST, PORT)
r = request(u, 'POST', body=LONG_BODY_PART)
t.eq(r.status_int, 200)
body = r.body_string()
t.eq(len(body), len(LONG_BODY_PART))
t.eq(body, LONG_BODY_PART)
def attach_img(db, tweet, url, attname):
res = restkit.request(url)
if res.status_int == 200:
# convert image in PNG
img = Image.open(BytesIO(res.body_string()))
out = BytesIO()
img.save(out, 'png')
out.seek(0)
# send to couchdb
db.put_attachment(tweet, out, "%s.png" % attname,
headers={'Transfer-Encoding': 'chunked'})
# create thumbnail if the attachment is a photo
if attname == 'photo':
ratio = 1
if img.size[0] > max_width:
ratio = max_width / float(img.size[0])
elif img.size[1] > max_height:
ratio = max_height / float(img.size[1])
w = img.size[0] * ratio
h = img.size[1] * ratio
thumb = img.resize((int(w), int(h)), Image.ANTIALIAS)
tout = BytesIO()
thumb.save(tout, 'png')
tout.seek(0)
# send it to couchdb
db.put_attachment(tweet, tout, "%s_thumb.png" % attname,
headers={'Transfer-Encoding': 'chunked'})
def child_sensorsRdf(self, ctx):
trig = ""
import restkit
for uri in ["http://bang:9069/graph", "http://bang:9070/graph"]:
trig += restkit.request(uri).body_string()
return trig
def readurl():
r = request('http://friendpaste.com/1ZSEoJeOarc3ULexzWOk5Y_633433316631/raw')
print "RESULT: %s: %s (%s)" % (u, r.status, len(r.body_string()))
# t = timeit.Timer(stmt=extract)
# print "%.2f s" % t.timeit(number=1)
def test_heartbeat(self):
# set initial heartbeat_value to current time
if not getattr(self, 'last_heartbeat_check', None):
self.last_heartbeat_check = 1
self.last_heartbeat_value = time()
# cache response value for 60 sec
if time() - self.last_heartbeat_check < 60:
return self.last_heartbeat_value
# ... or get from master
try:
r = request(self.slave_mode, timeout=5)
data = json.loads(r.body_string())
# log result
hv = data['heartbeat']
lag = time() - hv
logger.info("Master heartbeat %s lag %s min",
strftime('%H:%M:%S', localtime(hv)), int(lag / 60))
except Exception as e:
logger.error("Can't check heartbeat %s %s",
type(e).__name__, unicode(e))
# if request failed accept last successed value
data = {'heartbeat': self.last_heartbeat_value}
if 'index_names' in data:
self.names_db.update(data['index_names'])
self.last_heartbeat_check = time()
self.last_heartbeat_value = int(data.get('heartbeat') or 0)
return self.last_heartbeat_value
def req(url):
if settings.DEBUG:
print url
r = request(url)
if r.status_int not in (200, ):
raise Exception("can't reach url :%s\n%s" % (url, r.status))
return r
def test_002():
u = "http://%s:%s" % (HOST, PORT)
r = request(u, 'POST', body=LONG_BODY_PART)
t.eq(r.status_int, 200)
body = r.body_string()
t.eq(len(body), len(LONG_BODY_PART))
t.eq(body, LONG_BODY_PART)
def handle(self, *args, **options):
get_opt = options.get
url = get_opt('url')
action = get_opt('action')
verbosity = get_opt('verbosity')
if not url and not action:
self.stderr.write('Url or action is required')
return
if action:
url = action
response = restkit.request(url, method='OPTIONS')
if verbosity:
for header in response.response.headers:
self.stdout.write(header)
else:
targets = ('MS-ASProtocolVersions', 'MS-ASProtocolCommands')
self.stdout.write('\n')
for item in filter(lambda i: i[0] in targets,
response.response.headers):
self.stdout.write(item)
self.stdout.write('\n')
def _doRequest(self, action, method, body=None, headers=None):
url = urlparse.urljoin(self.uriBase, self.uriPath + '/' + action)
log.debug("URL via (%s): %s" % (method, url))
if headers is None:
headers = {}
headers['User-Agent'] = '@console.name@[email protected]@'
if method in ["GET", "HEAD", "DELETE", "POST", "PUT"]:
response = restkit.request(url,
method=method,
body=body,
headers=headers,
filters=[self.auth])
else:
raise restkit.errors.InvalidRequestMethod("Unsupported Method")
if response.status_int >= 400:
if response.status_int == 404:
raise restkit.errors.ResourceNotFound(response.body_string())
elif response.status_int == 401:
raise restkit.errors.Unauthorized("Unauthorized Request")
else:
raise restkit.errors.RequestFailed(response.body_string(),
response.status_int,
response)
# body_string() can only be called once, since it's a socket read, or
# will throw an AlreadyRead exception.
response.body = response.body_string()
return response
def get_resource_by_uri(path, content_type=None):
"""Get RESTful resource while nicely handling restkit exceptions"""
if " " in path:
raise AttributeError
if content_type:
headers = {'Accept': content_type}
else:
headers = {}
try:
res = request(path, headers=headers)
except (ResourceNotFound, Unauthorized, RequestFailed, RedirectLimit,
RequestError, InvalidUrl, ResponseError, ProxyError, BadStatusLine,
ParserError, UnexpectedEOF):
return None
except:
raise
if res is None:
return None
body = None
if res.status == '200 OK':
body = res.body_string()
if 'Content-Length' in res.headers:
content_length = res.headers['Content-Length']
elif 'content-length' in res.headers:
content_length = res.headers['content-length']
else:
print "Warning: Content length header not found!"
raise AttributeError
if not len(body) == int(content_length):
print "Warning: Body length not correct!"
raise AttributeError
return body
def search_twitter(db, q, since=0, concurrency=10):
base_url = "http://search.twitter.com/search.json"
params = {"q": q, "include_entities": "true", "result_type": "mixed"}
if since > 0:
params.update({"since_id": str(since)})
path = "?" + urllib.urlencode(params)
found = 0
while True:
resp = restkit.request(base_url + path)
with resp.body_stream() as stream:
res = json.load(stream)
results = res.get('results')
found += len(results)
for result in results:
process_tweet(db, result)
if res["page"] == 1 and res['max_id'] != since:
since = res['max_id']
# next page continue, else stop
if "next_page" in res:
path = res["next_page"]
else:
break
return (since, found)
def search_twitter(db, q, since=0, concurrency=10):
base_url = "http://search.twitter.com/search.json"
params = {"q": q, "include_entities": "true", "result_type": "mixed"}
if since > 0:
params.update({"since_id": str(since)})
path = "?" + urllib.urlencode(params)
found = 0
while True:
resp = restkit.request(base_url + path)
with resp.body_stream() as stream:
res = json.load(stream)
results = res.get('results')
found += len(results)
for result in results:
process_tweet(db, result)
if res["page"] == 1 and res['max_id'] != since:
since = res['max_id']
# next page continue, else stop
if "next_page" in res:
path = res["next_page"]
else:
break
return (since, found)
def _create_repository(self, username, password, repo_name):
'''Makes a REST call to Bitbucket to create the new remote repository
before we can push.'''
try:
resp = request('https://api.bitbucket.org/1.0/repositories/',
method='POST',
body=form_encode({'name': repo_name}),
filters=[BasicAuth(username, password)],
headers={
'Content-Type':
'application/x-www-form-urlencoded'
},
follow_redirect=True)
if resp.status_int == 401:
return CREATE_RESULT.UNAUTHORIZED
elif resp.status_int == 400:
return CREATE_RESULT.BAD_REQUEST
elif (resp.status_int // 100) != 2:
print 'Call failed:', resp.status_int, resp.body_string()
return CREATE_RESULT.REMOTE_ERROR
else:
return CREATE_RESULT.OK
except:
type_, message, tb = sys.exc_info()
print type_, message
print util.traceback_to_str(tb)
return CREATE_RESULT.UNKNOWN
def _doRequest(self, action, method, body=None, headers=None):
url = urlparse.urljoin(self.uriBase, self.uriPath + '/' + action)
log.debug("URL via (%s): %s" % (method, url))
if headers is None:
headers = {}
headers['User-Agent'] = '@console.name@[email protected]@'
if method in ["GET", "HEAD", "DELETE", "POST", "PUT"]:
response = restkit.request(url, method=method, body=body, headers=headers,
filters=[self.auth])
else:
raise restkit.errors.InvalidRequestMethod("Unsupported Method")
if response.status_int >= 400:
if response.status_int == 404:
raise restkit.errors.ResourceNotFound(response.body_string())
elif response.status_int == 401:
raise restkit.errors.Unauthorized("Unauthorized Request")
else:
raise restkit.errors.RequestFailed(response.body_string(), response.status_int, response)
# body_string() can only be called once, since it's a socket read, or
# will throw an AlreadyRead exception.
response.body = response.body_string()
return response
def readurl():
r = request(
'http://friendpaste.com/1ZSEoJeOarc3ULexzWOk5Y_633433316631/raw')
print "RESULT: %s: %s (%s)" % (u, r.status, len(r.body_string()))
# t = timeit.Timer(stmt=extract)
# print "%.2f s" % t.timeit(number=1)
def __call__(self, environ, start_response):
method = environ['REQUEST_METHOD']
if method not in self.allowed_methods:
start_response('403 Forbidden', ())
return ['']
if self.strip_script_name:
path_info = ''
else:
path_info = environ['SCRIPT_NAME']
path_info += environ['PATH_INFO']
query_string = environ['QUERY_STRING']
if query_string:
path_info += '?' + query_string
host_uri = self.extract_uri(environ)
uri = host_uri+path_info
new_headers = {}
for k, v in environ.items():
if k.startswith('HTTP_'):
k = k[5:].replace('_', '-').title()
new_headers[k] = v
for k, v in (('CONTENT_TYPE', None), ('CONTENT_LENGTH', '0')):
v = environ.get(k, None)
if v is not None:
new_headers[k.replace('_', '-').title()] = v
if new_headers.get('Content-Length', '0') == '-1':
raise ValueError(WEBOB_ERROR)
response = request(uri, method,
body=environ['wsgi.input'], headers=new_headers,
pool_instance=self.pool)
if 'location' in response:
headers = []
for k, v in response.headerslist:
if k == 'Location':
# rewrite location with a relative path. dont want to deal
# with complex url rebuild stuff
if v.startswith(host_uri):
v = v[len(host_uri):]
if self.strip_script_name:
v = environ['SCRIPT_NAME'] + v
headers.append((k, v))
else:
headers = response.headerslist
start_response(response.status, headers)
if 'content-length' in response and \
int(response['content-length']) <= MAX_BODY:
return [response.body]
return ResponseIter(response)
def postRequest(self):
"""
"""
url = 'https://' + self.ses.host + '/'
req = restkit.request(url, method=self.method, \
body=self.parameters, headers=self.headers)
body = req.body_string()
return {'status': req.status_int, 'body': body}
def getRequest(self):
"""
"""
query = urllib.urlencode(self.parameters)
url = 'https://' + self.ses.host + '/?' + query
req = restkit.request(url, method=self.method, headers=self.headers)
body = req.body_string()
return {'status': req.status_int, 'body': body}
def _fetch_data(from_date, to_date, page_size=20, first_page=1):
"""Fetch a chunk of vulndb"""
from_date = from_date.strftime("%Y-%m-%d")
to_date = to_date.strftime("%Y-%m-%d")
logger.info("Working on date range: {} - {}".format(from_date, to_date))
consumer = oauth2.Consumer(key=consumer_key, secret=consumer_secret)
# client = oauth2.Client(consumer)
# now get our request token
auth = OAuthFilter('*', consumer)
# initialize the page counter either at the first page or whatever page
# was requested
page_counter = first_page
finished = False
reply = dict()
reply['results'] = []
while not finished:
url = 'https://vulndb.cyberriskanalytics.com' + \
'/api/v1/vulnerabilities/find_by_date?' + \
'start_date=' + from_date + '&end_date=' + to_date + '&page=' + \
str(page_counter) + '&size=' + str(page_size) + \
'&date_type=updated_on' + \
'&nested=true'
logger.debug("Working on url: {} ".format(url))
resp = request(url, filters=[auth])
if resp.status_int == 404:
logger.warning("Could not find anything for the week " +
"begining: {}".format(from_date))
return
if resp.status_int != 200:
raise Exception("Invalid response {}.".format(resp['status']))
logger.debug("\tHTTP Response code: " + str(resp.status_int))
"""parse response and append to working set"""
page_reply = json.loads(resp.body_string())
logger.debug("Retrieving page {} of {}.".format(page_counter,
-(-page_reply['total_entries'] // page_size)))
if len(page_reply['results']) < page_size:
finished = True
reply['results'].extend(page_reply['results'])
reply['total_entries'] = page_reply['total_entries']
else:
page_counter += 1
reply['results'].extend(page_reply['results'])
logger.info("Returning {} out of {} results".format(str(len(
reply['results'])), str(reply['total_entries'])))
return reply
def fetchResource(self, url):
response = restkit.request(url=url, headers=self.proxyHeaders())
if not response.status.startswith('2'):
raise ValueError("request failed with %r" % response.status)
body = response.body_string()
log.info("got %s byte response, content-type %s", len(body),
response['headers'].get('content-type', None))
return body
def _fetch_data(from_date, to_date, page_size=20, first_page=1):
"""Fetch a chunk of vulndb"""
from_date = from_date.strftime("%Y-%m-%d")
to_date = to_date.strftime("%Y-%m-%d")
logger.info("Working on date range: {} - {}".format(from_date, to_date))
consumer = oauth2.Consumer(key=consumer_key, secret=consumer_secret)
# client = oauth2.Client(consumer)
# now get our request token
auth = OAuthFilter('*', consumer)
# initialize the page counter either at the first page or whatever page
# was requested
page_counter = first_page
finished = False
reply = dict()
reply['results'] = []
while not finished:
url = 'https://vulndb.cyberriskanalytics.com' + \
'/api/v1/vulnerabilities/find_by_date?' + \
'start_date=' + from_date + '&end_date=' + to_date + '&page=' + \
str(page_counter) + '&size=' + str(page_size) + \
'&date_type=updated_on' + \
'&nested=true'
logger.debug("Working on url: {} ".format(url))
resp = request(url, filters=[auth])
if resp.status_int == 404:
logger.warning("Could not find anything for the week " +
"begining: {}".format(from_date))
return
if resp.status_int != 200:
raise Exception("Invalid response {}.".format(resp['status']))
logger.debug("\tHTTP Response code: " + str(resp.status_int))
"""parse response and append to working set"""
page_reply = json.loads(resp.body_string())
logger.debug("Retrieving page {} of {}.".format(
page_counter, -(-page_reply['total_entries'] // page_size)))
if len(page_reply['results']) < page_size:
finished = True
reply['results'].extend(page_reply['results'])
reply['total_entries'] = page_reply['total_entries']
else:
page_counter += 1
reply['results'].extend(page_reply['results'])
logger.info("Returning {} out of {} results".format(
str(len(reply['results'])), str(reply['total_entries'])))
return reply
def __call__(self, environ, start_response):
method = environ['REQUEST_METHOD']
if method not in self.allowed_methods:
start_response('403 Forbidden', ())
return ['']
if self.strip_script_name:
path_info = ''
else:
path_info = environ['SCRIPT_NAME']
path_info += environ['PATH_INFO']
query_string = environ['QUERY_STRING']
if query_string:
path_info += '?' + query_string
host_uri = self.extract_uri(environ)
uri = host_uri + path_info
new_headers = {}
for k, v in environ.items():
if k.startswith('HTTP_'):
k = k[5:].replace('_', '-').title()
new_headers[k] = v
for k, v in (('CONTENT_TYPE', None), ('CONTENT_LENGTH', '0')):
v = environ.get(k, None)
if v is not None:
new_headers[k.replace('_', '-').title()] = v
if new_headers.get('Content-Length', '0') == '-1':
raise ValueError(WEBOB_ERROR)
response = request(uri,
method,
body=environ['wsgi.input'],
headers=new_headers,
pool_instance=self.pool)
if 'location' in response:
headers = []
for k, v in response.headerslist:
if k.lower() == 'location':
# rewrite location with a relative path. dont want to deal
# with complex url rebuild stuff
if v.startswith(host_uri):
v = v[len(host_uri):]
if self.strip_script_name:
v = environ['SCRIPT_NAME'] + v
headers.append((k, v))
else:
headers = response.headerslist
start_response(response.status, headers)
return response.body_stream()
def all_roles(self,token_encode,options={}):
try:
gurl = 'https://authz-%s-conjur.herokuapp.com' % (options["stack"])
headers = {"Authorization":token_encode}
url = "/%s/roles/user/%s/?all" % (options["account"],options["username"])
r = request(gurl+url, method='GET', headers=headers)
jsonenv = json.loads(r.body_string())
return jsonenv
except Exception,exc:
print "Error roles: %s" % (exc)
def test_003(o, u, b):
r = request(u, "POST", body=b, filters=[o],
headers={"Content-type": "application/x-www-form-urlencoded"})
import sys
print >>sys.stderr, r.body_string()
t.eq(r.status_int, 200)
# Because this is a POST and an application/x-www-form-urlencoded, the OAuth
# can include the OAuth parameters directly into the body of the form, however
# it MUST NOT include the 'oauth_body_hash' parameter in these circumstances.
t.isnotin("oauth_body_hash", r.request.body)
def count_user_commits(user):
r = request('https://api.github.com/users/%s/repos' % user)
repos = json.loads(r.body_string())
for repo in repos:
if repo['fork'] is True:
# skip it
continue
n = count_repo_commits(repo['url'] + '/commits')
yield (repo['name'], n)
def fetchResource(self, url):
response = restkit.request(url=url,
headers=self.proxyHeaders())
if not response.status.startswith('2'):
raise ValueError("request failed with %r" % response.status)
body = response.body_string()
log.info("got %s byte response, content-type %s", len(body),
response['headers'].get('content-type', None))
return body
def __call__(self, environ, start_response):
method = environ["REQUEST_METHOD"]
if method not in self.allowed_methods:
start_response("403 Forbidden", ())
return [""]
if self.strip_script_name:
path_info = ""
else:
path_info = environ["SCRIPT_NAME"]
path_info += environ["PATH_INFO"]
query_string = environ["QUERY_STRING"]
if query_string:
path_info += "?" + query_string
host_uri = self.extract_uri(environ)
uri = host_uri + path_info
new_headers = {}
for k, v in environ.items():
if k.startswith("HTTP_"):
k = k[5:].replace("_", "-").title()
new_headers[k] = v
for k, v in (("CONTENT_TYPE", None), ("CONTENT_LENGTH", "0")):
v = environ.get(k, None)
if v is not None:
new_headers[k.replace("_", "-").title()] = v
if new_headers.get("Content-Length", "0") == "-1":
raise ValueError(WEBOB_ERROR)
response = request(uri, method, body=environ["wsgi.input"], headers=new_headers, pool_instance=self.pool)
if "location" in response:
headers = []
for k, v in response.headerslist:
if k == "Location":
# rewrite location with a relative path. dont want to deal
# with complex url rebuild stuff
if v.startswith(host_uri):
v = v[len(host_uri) :]
if self.strip_script_name:
v = environ["SCRIPT_NAME"] + v
headers.append((k, v))
else:
headers = response.headerslist
start_response(response.status, headers)
if "content-length" in response and int(response["content-length"]) <= MAX_BODY:
return [response.body]
return ResponseIter(response)
def export(self,
project_id,
language_code,
file_type='po',
filters=None,
tags=None,
local_file=None):
"""
Return terms / translations
filters - filter by self._filter_by
tags - filter results by tags;
local_file - save content into it. If None, save content into
random temp file.
>>> tags = 'name-of-tag'
>>> tags = ["name-of-tag"]
>>> tags = ["name-of-tag", "name-of-another-tag"]
>>> filters = 'translated'
>>> filters = ["translated"]
>>> filters = ["translated", "not_fuzzy"]
"""
if file_type not in self.FILE_TYPES:
raise POEditorArgsException(u'content_type: file format {}'.format(
self.FILE_TYPES))
if filters and filters not in self.FILTER_BY:
raise POEditorArgsException(
u"filters - filter results by {}".format(self.FILTER_BY))
data = self._run(action="export",
id=project_id,
language=language_code,
type=file_type,
filters=filters,
tags=tags)
file_url = data['item']
# Download file content:
res = request(file_url)
if not local_file:
tmp_file = tempfile.NamedTemporaryFile(
delete=False, suffix='.{}'.format(file_type))
tmp_file.close()
local_file = tmp_file.name
with open(local_file, 'w+b') as po_file:
with res.body_stream() as body:
while True:
data = body.read(1024)
if not data:
break
po_file.write(data)
return file_url, local_file
def test_004():
u = "http://%s:%s/multipart2" % (HOST, PORT)
fn = os.path.join(os.path.dirname(__file__), "1M")
f = open(fn, 'rb')
l = int(os.fstat(f.fileno())[6])
b = {'a': 'aa', 'b': ['bb', 'éàù@'], 'f': f}
h = {'content-type': "multipart/form-data"}
body, headers = multipart_form_encode(b, h, uuid.uuid4().hex)
r = request(u, method='POST', body=body, headers=headers)
t.eq(r.status_int, 200)
t.eq(int(r.body_string()), l)
def guess_language(text):
if not restkit:
return None
if isinstance(text, unicode):
text = text.encode('utf-8')
url = 'http://www.google.com/uds/GlangDetect?'
url += urllib.urlencode(dict(v='1.0', q=text))
resp = restkit.request(url)
data = resp.body_string()
data = json.loads(data)
return data.get('responseData', {}).get('language', None)
def test_004():
u = "http://%s:%s/multipart2" % (HOST, PORT)
fn = os.path.join(os.path.dirname(__file__), "1M")
f = open(fn, 'rb')
l = int(os.fstat(f.fileno())[6])
b = {'a': 'aa', 'b': ['bb', six.b('éàù@')], 'f': f}
h = {'content-type': "multipart/form-data"}
body, headers = multipart_form_encode(b, h, uuid.uuid4().hex)
r = request(u, method='POST', body=body, headers=headers)
t.eq(r.status_int, 200)
t.eq(int(r.body_string()), l)
def test_006():
u = "http://%s:%s/multipart4" % (HOST, PORT)
fn = os.path.join(os.path.dirname(__file__), "1M")
f = open(fn, 'rb')
content = f.read()
f.seek(0)
b = {'a': 'aa', 'b': six.b('éàù@'), 'f': f}
h = {'content-type': "multipart/form-data"}
body, headers = multipart_form_encode(b, h, uuid.uuid4().hex)
r = request(u, method='POST', body=body, headers=headers)
t.eq(r.status_int, 200)
t.eq(r.body_string(), content)
def test_007():
import StringIO
u = "http://%s:%s/multipart4" % (HOST, PORT)
content = 'éàù@'
f = StringIO.StringIO('éàù@')
f.name = 'test.txt'
b = {'a': 'aa', 'b': 'éàù@', 'f': f}
h = {'content-type': "multipart/form-data"}
body, headers = multipart_form_encode(b, h, uuid.uuid4().hex)
r = request(u, method='POST', body=body, headers=headers)
t.eq(r.status_int, 200)
t.eq(r.body_string(), content)
def test_007():
from six import BytesIO as StringIO
u = "http://%s:%s/multipart4" % (HOST, PORT)
content = six.b('éàù@')
f = StringIO(six.b('éàù@'))
f.name = 'test.txt'
b = {'a': 'aa', 'b': six.b('éàù@'), 'f': f}
h = {'content-type': "multipart/form-data"}
body, headers = multipart_form_encode(b, h, uuid.uuid4().hex)
r = request(u, method='POST', body=body, headers=headers)
t.eq(r.status_int, 200)
t.eq(r.body_string(), content)
def test_006():
u = "http://%s:%s/multipart4" % (HOST, PORT)
fn = os.path.join(os.path.dirname(__file__), "1M")
f = open(fn, 'rb')
content = f.read()
f.seek(0)
b = {'a': 'aa', 'b': 'éàù@', 'f': f}
h = {'content-type': "multipart/form-data"}
body, headers = multipart_form_encode(b, h, uuid.uuid4().hex)
r = request(u, method='POST', body=body, headers=headers)
t.eq(r.status_int, 200)
t.eq(r.body_string(), content)
def syncServiceCall(name, photoUri, foafUser, **moreParams):
if not photoUri:
raise ValueError("no uri to %s service call" % name)
t1 = time.time()
params = {'uri': photoUri}
params.update(moreParams)
url = '%s?%s' % (networking.serviceUrl(name), urllib.urlencode(params))
response = restkit.request(url=url, headers={'x-foaf-agent': foafUser})
log.info("service call %r in %.01f ms", name, 1000 * (time.time() - t1))
if response.status_int != 200:
raise ValueError("in service call %s" % url)
return response.body_string()
def syncServiceCall(name, photoUri, foafUser, **moreParams):
if not photoUri:
raise ValueError("no uri to %s service call" % name)
t1 = time.time()
params = {'uri' : photoUri}
params.update(moreParams)
url = '%s?%s' % (networking.serviceUrl(name), urllib.urlencode(params))
response = restkit.request(url=url, headers={'x-foaf-agent' : foafUser})
log.info("service call %r in %.01f ms", name, 1000 * (time.time() - t1))
if response.status_int != 200:
raise ValueError("in service call %s" % url)
return response.body_string()
def getRequest(self):
"""
"""
query = urllib.urlencode(self.parameters)
url = 'https://' + self.ses.host + '/?' + query
req = restkit.request(url, method=self.method, headers=self.headers)
body = req.body_string()
return {
'status': req.status_int,
'body': body
}
def postTwitter(graph, openid, msg):
"""
openid is an RDF node that foaf:holdsAccount which is a mb:TwitterAccount
"""
oauthFilter = makeOauthFilter(graph, openid)
resp = restkit.request(
method="POST",
url="http://api.twitter.com/1.1/statuses/update.json",
filters=[oauthFilter],
body={'status' : msg})
if resp.status_int != 200:
raise ValueError("%s: %s" % (resp.status, resp.body_string()))
def postRequest(self):
"""
"""
url = 'https://' + self.ses.host + '/'
req = restkit.request(url, method=self.method, \
body=self.parameters, headers=self.headers)
body = req.body_string()
return {
'status': req.status_int,
'body': body
}
def add_file(self, article_id, file_path):
""" Upload a file with path 'file_path' to an article with a given article_id. Return the ID of the file.
This code is based on the example from the Figshare API documentation: https://docs.figshare.com/api/upload_example/"""
file_name = os.path.basename(file_path)
# Get file info
file_info = {}
hash = hashlib.md5()
with open(file_path, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash.update(chunk)
file_info['md5'] = hash.hexdigest()
file_info['name'] = file_name
file_info['size'] = os.path.getsize(file_path)
# Create file object
payload = json.dumps(file_info)
response = self.post('/account/articles/{}/files'.format(article_id), headers=self.get_headers(token=self.token), payload=payload)
response = json.loads(response.body_string())
file_location = response["location"]
# Get the file info, in particular the upload URL
response = request(file_location, headers=self.get_headers(token=self.token))
upload_url = json.loads(response.body_string())["upload_url"]
# Upload the file
response = request(upload_url, headers=self.get_headers(token=self.token))
response = json.loads(response.body_string())
parts = response["parts"]
with open(file_path, 'rb') as file_input:
for part in parts:
size = part['endOffset'] - part['startOffset'] + 1
response = request('{0}/{1}'.format(upload_url, part["partNo"]), method='PUT', body=file_input.read(size))
response = request(file_location, method='POST', headers=self.get_headers(token=self.token))
file_id = int(file_location.split("/")[-1])
return file_id
def export(self, project_id, language_code, file_type='po', filters=None,
tags=None, local_file=None):
"""
Return terms / translations
filters - filter by self._filter_by
tags - filter results by tags;
local_file - save content into it. If None, save content into
random temp file.
>>> tags = 'name-of-tag'
>>> tags = ["name-of-tag"]
>>> tags = ["name-of-tag", "name-of-another-tag"]
>>> filters = 'translated'
>>> filters = ["translated"]
>>> filters = ["translated", "not_fuzzy"]
"""
if file_type not in self.FILE_TYPES:
raise POEditorArgsException(
u'content_type: file format {}'.format(self.FILE_TYPES))
if filters and filters not in self.FILTER_BY:
raise POEditorArgsException(
u"filters - filter results by {}".format(self.FILTER_BY))
data = self._run(
action="export",
id=project_id,
language=language_code,
type=file_type,
filters=filters,
tags=tags
)
file_url = data['item']
# Download file content:
res = request(file_url)
if not local_file:
tmp_file = tempfile.NamedTemporaryFile(
delete=False, suffix='.{}'.format(file_type))
tmp_file.close()
local_file = tmp_file.name
with open(local_file, 'w+b') as po_file:
with res.body_stream() as body:
while True:
data = body.read(1024)
if not data:
break
po_file.write(data)
return file_url, local_file
def token_from_code(self, code):
"""Step 2"""
headers = dict()
headers['Content-Type'] = 'application/x-www-form-urlencoded'
resp = request(URL_STEP2, 'POST', body=self._step2_body(code),
headers=headers)
body = resp.body_string()
result = json.loads(body)
if u'error' in result:
raise SoundcloudException(result[u'error'])
# XXX Error handling is missing
self.token = result[u'access_token']
return self.token
def token_from_code(self, code):
"""Step 2"""
headers = dict()
headers['Content-Type'] = 'application/x-www-form-urlencoded'
resp = request(URL_STEP2, 'POST', body=self._step2_body(code),
headers=headers)
body = resp.body_string()
result = json.loads(body)
if u'error' in result:
raise SoundcloudException(result[u'error'])
# XXX Error handling is missing
self.token = result[u'access_token']
return self.token
def call_upload_new_file(self, id, file):
'''
Upload a file and associate it with an article.
:type id: int
:param id: the id of the article
:type file: str
:param file: the file to upload
:return: the upload location
:rtype: ArticleLocation
'''
payload = json.dumps(create_fileupload_dict(file))
response = self.post('/account/articles/{}/files'.format(id),
headers=get_headers(token=self.token),
payload=payload)
loc = ArticleLocation(**json.loads(response.body_string()))
response = request(loc.location, headers=get_headers(token=self.token))
article_file = ArticleFile(**json.loads(response.body_string()))
# upload_url = '{0}/{1}'.format(article_file.upload_url,
# article_file.upload_token)
upload_url = article_file.upload_url
response = request(upload_url)
article_file_upload_status = ArticleFileUploadStatus(
**json.loads(response.body_string()))
with open(file, 'rb') as file_input:
for part in article_file_upload_status.parts:
size = part['endOffset'] - part['startOffset'] + 1
response = request('{0}/{1}'.format(upload_url, part.partNo),
method='PUT',
body=file_input.read(size))
response = request(loc.location,
method='POST',
headers=get_headers(token=self.token))
return loc
def test_003(o, u, b):
r = request(u,
"POST",
body=b,
filters=[o],
headers={"Content-type": "application/x-www-form-urlencoded"})
import sys
print(r.body_string(), file=sys.stderr)
t.eq(r.status_int, 200)
# Because this is a POST and an application/x-www-form-urlencoded, the OAuth
# can include the OAuth parameters directly into the body of the form, however
# it MUST NOT include the 'oauth_body_hash' parameter in these circumstances.
t.isnotin("oauth_body_hash", r.request.body)
def count_repo_commits(commits_url, _acc=0):
r = request(commits_url)
commits = json.loads(r.body_string())
n = len(commits)
if n == 0:
return _acc
link = r.headers.get('link')
if link is None:
return _acc + n
next_url = find_next(r.headers['link'])
if next_url is None:
return _acc + n
# try to be tail recursive, even when it doesn't matter in CPython
return count_repo_commits(next_url, _acc + n)
def create_graph_image(graph_data, image_file):
""" Given a formatted blob of graphviz chart data[1], make the actual request to Google
and store the resulting image to disk.
[1]: http://code.google.com/apis/chart/docs/gallery/graphviz.html
"""
chart_url = GOOGLE_CHART_URL + 'cht=gv&chl=digraph{%s}' % ';'.join(graph_data)
g = request(chart_url)
print('Writing to ' + image_file)
image = open(image_file, 'w+')
image.write(g.body_stream().read())
image.close()
return image_file
def call_upload_new_file(self, id, file):
'''
Upload a file and associate it with an article.
:type id: int
:param id: the id of the article
:type file: str
:param file: the file to upload
:return: the upload location
:rtype: ArticleLocation
'''
payload = json.dumps(create_fileupload_dict(file))
response = self.post('/account/articles/{}/files'.format(id),
headers=get_headers(token=self.token), payload=payload)
loc = ArticleLocation(**json.loads(response.body_string()))
response = request(loc.location, headers=get_headers(token=self.token))
article_file = ArticleFile(**json.loads(response.body_string()))
# upload_url = '{0}/{1}'.format(article_file.upload_url,
# article_file.upload_token)
upload_url = article_file.upload_url
response = request(upload_url)
article_file_upload_status = ArticleFileUploadStatus(
**json.loads(response.body_string()))
with open(file, 'rb') as file_input:
for part in article_file_upload_status.parts:
size = part['endOffset'] - part['startOffset'] + 1
response = request(
'{0}/{1}'.format(upload_url, part.partNo), method='PUT', body=file_input.read(size))
response = request(loc.location, method='POST',
headers=get_headers(token=self.token))
return loc
def getPageTitleNow(self, uri):
try:
response = restkit.request(uri, timeout=1, follow_redirect=True,
headers={
'user-agent':
'link title checker - [email protected]'
})
if not response.status.startswith('2'):
raise CantGetTitle("(got %s)" % response.status)
root = lxml.html.soupparser.fromstring(
response.body_string())
for title in root.cssselect("title"):
return title.text
except restkit.RequestError:
raise CantGetTitle("(error requesting title from site)")
def fetch(url):
"Fetches `url` and writes the result to redis"
#logging.info('start %s', url)
data = dict(url=url, fetched_at=datetime.datetime.now().isoformat())
try:
req = request(url)
except RequestError:
# invalid url
pass
else:
data.update(headers=dict(req.headers),
body=req.body_string().strip(),
status=req.status_int)
r.lpush('fetched', pickle.dumps(data))
r.lrem('new_backup', url)
logging.info('fetched %s', url)
return data
def get_file(self, tender, url, access_token=None):
parsed_url = urlparse(url)
headers = {}
if access_token:
headers = {'X-Access-Token': access_token}
headers.update(self.headers)
response_item = self.get(parsed_url.path,
headers=headers,
params_dict=parse_qs(parsed_url.query))
if response_item.status_int == 302:
response_obj = request(response_item.headers['location'])
if response_obj.status_int == 200:
return response_obj.body_string(), \
response_obj.headers['Content-Disposition'] \
.split(";")[1].split('"')[1]
raise InvalidResponse
