def get_apache_process(url = "http://localhost/server-status", re_status = "W|G", pid = 0, request_pattern = "", notrequest_pattern = "", mintime = 0, re_pstatus = "D|R|S|T|W|X|Z"):
status_pattern = re.compile(re_status)
pstatus_pattern = re.compile(re_pstatus)
#num_pattern = re.compile("[0-9]+")
buffer=StringIO.StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.perform()
curl.close()
xpath = etree.HTML(buffer.getvalue())
tables = xpath.xpath('//table[1]/tr[position()>1]')
found = False
processes_list = {}
for tr in tables:
tds = tr.xpath('td')
b = tds[3].xpath('b')
if len(b)>0:
cstatus = b[0].text.strip()
else:
cstatus = tds[3].text.strip()
try:
for line in open("/proc/%d/status" % int(tds[1].text)).readlines():
if line.startswith("State:"):
pstatus = line.split(":",1)[1].strip().split(' ')[0]
except ValueError:
pstatus = '=' # Means nothing, just not to match the regex
except IOError:
# Too bad process do not exist any more
pstatus = '='
if status_pattern.match(cstatus) and pstatus_pattern.match(pstatus):
if int(tds[5].text) < mintime: # Mintime too low
continue
if request_pattern and not re.search(request_pattern, tds[12].text): # Not the URL we are looking for
continue
if notrequest_pattern and re.search(notrequest_pattern, tds[12].text): # Not the URL we are looking for
continue
if tds[10].text == '127.0.0.1' and re.search('^GET /server-status', tds[12].text):
continue
if pid > 0 and int(tds[1].text) == pid:
found = True
processes_list[int(tds[1].text)] = { 'pid': int(tds[1].text), 'time': tds[5].text, 'status': cstatus, 'request': tds[12].text,
'client': tds[10].text, 'vhost': tds[11].text, 'pstatus': pstatus }
elif pid == 0:
processes_list[int(tds[1].text)] = { 'pid': int(tds[1].text), 'time': tds[5].text, 'status': cstatus, 'request': tds[12].text,
'client': tds[10].text, 'vhost': tds[11].text, 'pstatus': pstatus }
if pid > 0 and not found:
print("PID {0} not found".format(pid))
return False
return processes_list
def like():
likecount = 0
sleepcount = 0
likelimit = int(arguments['--likelimit'])
if not (arguments['--maxlikelimit'] == 'unlimited'):
maxlikelimit = int(arguments['--maxlikelimit'])
else:
maxlikelimit = 0
print "Hashtag dictionnary: "+str(hashtags[arguments['--hashtags']])
print "Will try to like "+str(arguments['--likelimit'])+" of each of these tags until "+str(arguments['--maxlikelimit'])+"."
print "----"
for tag in hashtags[arguments['--hashtags']]:
print ": Liking: %s" % bold(tag)
hashtaglikes = 0
nextpage = "http://web.stagram.com/tag/"+tag+"/?vm=list"
#enter hashtag like loop
while nextpage != False and (likelimit == 0 or (likelimit > 0 and hashtaglikes < likelimit)):
buf = cStringIO.StringIO()
c = pycurl.Curl()
c.setopt(pycurl.URL, nextpage)
c.setopt(pycurl.COOKIEFILE, "instanax.txt")
c.setopt(pycurl.COOKIEJAR, "instanax.txt")
c.setopt(pycurl.WRITEFUNCTION, buf.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.ENCODING, "")
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
useragent = UA()
c.setopt(pycurl.USERAGENT, useragent)
c.perform()
curlData = buf.getvalue()
buf.close()
nextpagelink = re.findall(ur"<a href=\"([^\"]*)\" rel=\"next\">Earlier<\/a>",curlData)
if len(nextpagelink)>0:
nextpage = "http://web.stagram.com"+nextpagelink[0]
else:
nextpage = False
likedata = re.findall(ur"<span class=\"like_button\" id=\"like_button_([^\"]*)\">",curlData)
if len(likedata)>0:
for imageid in likedata:
if likelimit > 0 and hashtaglikes >= likelimit:
break
repeat = True
while repeat:
randomint = random.randint(1000,9999)
postdata = 'pk='+imageid+'&t='+str(randomint)
buf = cStringIO.StringIO()
c = pycurl.Curl()
c.setopt(pycurl.URL, "http://web.stagram.com/do_like/")
c.setopt(pycurl.COOKIEFILE, "instanax.txt")
c.setopt(pycurl.COOKIEJAR, "instanax.txt")
c.setopt(pycurl.WRITEFUNCTION, buf.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.ENCODING, "")
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
useragent = UA()
c.setopt(pycurl.USERAGENT, useragent)
c.setopt(pycurl.POST, 1)
c.setopt(pycurl.POSTFIELDS, postdata)
c.setopt(pycurl.POSTFIELDSIZE, len(postdata))
#c.setopt(pycurl.VERBOSE, True)
c.perform()
postData = buf.getvalue()
buf.close()
if postData == '''{"status":"OK","message":"LIKED"}''':
likecount += 1
hashtaglikes += 1
# print " - You liked #"+tag+" image "+imageid+"! Like count: "+str(likecount)+" (http://web.stagram.com/p/"+imageid+")"
likemsg = " - You liked http://web.stagram.com/p/"+imageid+"! "
print likemsg.ljust(68, ' ')+"Like count: "+str(likecount).rjust(3,' ')
repeat = False
sleepcount = 0
st = 0
if (arguments['--sleeptimer'] == 'auto'):
st = random.uniform(1, 10)
time.sleep(st)
elif arguments['--sleeptimer'] > 0:
st = float(arguments['--sleeptimer'])
time.sleep(st)
print " ...slept for "+str(st)+" seconds"
else:
sleepcount += 1
print "Your account has been rate limited. Sleeping on "+tag+" for "+str(sleepcount*10)+" minutes. Liked "+str(likecount)+" photos until now..."
time.sleep(10*60)
if maxlikelimit > 0 and maxlikelimit <= likecount:
print ": Max Like Limit reached, exiting"
return
#author:http://book.51cto.com/art/201411/456735.htm
# -*- coding: utf-8 -*-
import os,sys
import time
import sys
import pycurl
print "URL: http://www.baidu.com " #探测的目标URL
URL=raw_input('please you input URL: ')
#URL="http://www.baidu.com" #探测的目标URL
c = pycurl.Curl() #创建一个Curl对象
c.setopt(pycurl.URL, URL) #定义请求的URL常量 设置为0则不等待
c.setopt(pycurl.CONNECTTIMEOUT, 30) #定义请求连接的等待时间
c.setopt(pycurl.TIMEOUT, 30) #定义请求超时时间
c.setopt(pycurl.NOPROGRESS, 1) #屏蔽下载进度条 非 0则屏蔽
c.setopt(pycurl.FORBID_REUSE, 1) #完成交互后强制断开连接，不重用
c.setopt(pycurl.MAXREDIRS, 1) #指定HTTP重定向的最大数为1
c.setopt(pycurl.DNS_CACHE_TIMEOUT,30) #设置保存DNS信息的时间为30秒 默认120秒
#创建一个文件对象，以”wb”方式打开，用来存储返回的http头部及页面内容
indexfile = open(os.path.dirname(os.path.realpath(__file__))+"/content.txt", "wb")
c.setopt(pycurl.WRITEHEADER, indexfile) #将返回的HTTP HEADER定向到indexfile文件对象
c.setopt(pycurl.WRITEDATA, indexfile) #将返回的HTML内容定向到indexfile文件对象
try:
c.perform() # 执行curl命令
except Exception,e:
#print e
def test_ok(self):
curl = pycurl.Curl()
self.assertEqual(0, curl.SEEKFUNC_OK)
curl.close()
def test_cantseek(self):
curl = pycurl.Curl()
self.assertEqual(2, curl.SEEKFUNC_CANTSEEK)
curl.close()
#!/usr/bin/env python
from config import config
import pycurl, cStringIO
buf = cStringIO.StringIO()
fields = {'token': config['api_token'], 'content': 'user', 'format': 'json'}
ch = pycurl.Curl()
ch.setopt(ch.URL, config['api_url'])
ch.setopt(ch.HTTPPOST, fields.items())
ch.setopt(ch.WRITEFUNCTION, buf.write)
ch.perform()
ch.close()
print buf.getvalue()
buf.close()
def rs2_webadmin_worker(delayed_queue: mp.Queue, instant_queue: mp.Queue,
log_queue: mp.Queue, login_url: str, chat_url: str,
username: str, password: str):
mplogger.worker_configurer(log_queue)
# noinspection PyShadowingNames
logger = logging.getLogger(__file__ + ":" + __name__)
logger.info("Starting rs2_webadmin_worker pid: %s", os.getpid())
auth_data = authenticate(login_url, username, password)
t = time.time()
last_ping_time = time.time()
while True:
try:
if auth_timed_out(t, auth_data.timeout):
logger.info("rs2_webadmin_worker(): Re-authenticating")
auth_data = authenticate(login_url, username, password)
t = time.time()
c = pycurl.Curl()
latest_sessionid = find_sessionid(HEADERS)
logger.info("rs2_webadmin_worker(): latest sessionid: %s",
latest_sessionid)
if latest_sessionid:
auth_data.sessionid = latest_sessionid
resp = get_messages(c, chat_url, auth_data.sessionid,
auth_data.authcred, auth_data.timeout)
encoding = read_encoding(HEADERS, -1)
logger.info("rs2_webadmin_worker(): Encoding from headers: %s",
encoding)
parsed_html = BeautifulSoup(resp.decode(encoding),
features="html.parser")
logger.debug("rs2_webadmin_worker(): Raw HTML response: %s",
parsed_html)
chat_message_divs = parsed_html.find_all(
"div", attrs={"class": "chatmessage"})
chat_notice_divs = parsed_html.find_all(
"div", attrs={"class": "chatnotice"})
logger.info(
"rs2_webadmin_worker(): Got %s 'class=chatmessage' divs from WebAdmin",
len(chat_message_divs))
logger.info(
"rs2_webadmin_worker(): Got %s 'class=chatnotice' divs from WebAdmin",
len(chat_notice_divs))
for i, div in enumerate(chat_message_divs):
tc, tn, name, msg = parse_chat_message_div(div)
sanitized_msg = msg.text.replace("@", "[at]")
if msg.text.lower().lstrip().startswith(PING_ADMIN):
logger.info("rs2_webadmin_worker(): detected !admin ping")
instant_queue.put(div)
logger.info(
"rs2_webadmin_worker(): Enqueued div no. %s in instant queue",
i)
ping_div = BeautifulSoup(f"""<div class="chatmessage">
<span class="teamcolor" style="background: #E54927;"> </span>
<span class="username">__RADIOMAN__</span>:
<span class="message">__ADMIN SUMMONED INGAME__ by: {name}! {PING_HC} {PING_PO} '{sanitized_msg}'</span>
</div>""",
features="html.parser")
if time.time() > (last_ping_time + (60 * 15)):
instant_queue.put(ping_div)
last_ping_time = time.time()
logger.info(
"rs2_webadmin_worker(): Enqueued ping_div no. %s in instant queue",
i)
else:
delayed_queue.put(
(div, time.time(), POST_CHAT_MESSAGE_DELAY_SECONDS))
logger.info(
"rs2_webadmin_worker(): Enqueued div no. %s in delayed queue",
i)
c.close()
time.sleep(5)
except pycurl.error as pe:
logger.error("error: %s", pe)
logger.error("retrying after: 50 ms")
time.sleep(0.05)
def _pycurl_post(self,
url,
json=None,
data=None,
username="",
password="",
headers={},
timeout=30):
"""This function will POST to the url endpoint using pycurl. returning
an AdyenResult object on 200 HTTP responce. Either json or data has to
be provided. If username and password are provided, basic auth will be
used.
Args:
url (str): url to send the POST
json (dict, optional): Dict of the JSON to POST
data (dict, optional): Dict, presumed flat structure of key/value of
request to place
username (str, optionl): Username for basic auth. Must be included
as part of password.
password (str, optional): Password for basic auth. Must be included
as part of username.
headers (dict, optional): Key/Value pairs of headers to include
timeout (int, optional): Default 30. Timeout for the request.
Returns:
str: Raw response received
str: Raw request placed
int: HTTP status code, eg 200,404,401
dict: Key/Value pairs of the headers received.
"""
response_headers={}
def handle_header(header_line):
header_line = header_line.decode('iso-8859-1')
if ':' in header_line:
name, value = header_line.split(':', 1)
name = name.strip()
value = value.strip()
response_headers[name] = value
curl = pycurl.Curl()
curl.setopt(curl.URL, url)
if sys.version_info[0] >= 3:
stringbuffer = BytesIO()
else:
stringbuffer = StringIO()
#stringbuffer = StringIO()
curl.setopt(curl.WRITEDATA, stringbuffer)
# Add User-Agent header to request so that the request can be identified as coming
# from the Adyen Python library.
headers['User-Agent'] = self.user_agent
# Convert the header dict to formatted array as pycurl needs.
if sys.version_info[0] >= 3:
header_list = ["%s:%s" % (k, v) for k, v in headers.items()]
else:
header_list = ["%s:%s" % (k, v) for k, v in headers.iteritems()]
#Ensure proper content-type when adding headers
if json:
header_list.append("Content-Type:application/json")
curl.setopt(pycurl.HTTPHEADER, header_list)
# Return regular dict instead of JSON encoded dict for request:
raw_store = json
# Set the request body.
raw_request = json_lib.dumps(json) if json else urlencode(data)
curl.setopt(curl.POSTFIELDS, raw_request)
if username and password:
curl.setopt(curl.USERPWD, '%s:%s' % (username, password))
curl.setopt(curl.TIMEOUT, timeout)
curl.perform()
# Grab the response content
result = stringbuffer.getvalue()
status_code = curl.getinfo(curl.RESPONSE_CODE)
curl.close()
# Return regular dict instead of JSON encoded dict for request:
raw_request = raw_store
return result, raw_request, status_code, response_headers
def send(url, sendData=None):
"""Connect to url and return the result as stringIO
:arg url: the url where the request will be sent
:kwarg sendData: do a post-request when "sendData" is given.
Returns the result as stringIO object.
"""
connect_retries = 10
try_counter = connect_retries
timeout = 120
if CFG.is_initialized() and CFG.has_key('TIMEOUT'):
timeout = CFG.TIMEOUT
curl = pycurl.Curl()
curl.setopt(pycurl.CONNECTTIMEOUT, timeout)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
curl.setopt(pycurl.VERBOSE, True)
proxy_url, proxy_user, proxy_pass = get_proxy(url)
if proxy_url:
curl.setopt(pycurl.PROXY, proxy_url)
log_debug(2, "Connect to %s" % url)
if sendData is not None:
curl.setopt(pycurl.POSTFIELDS, sendData)
if (CFG.is_initialized() and CFG.has_key('DISABLE_EXPECT')
and CFG.DISABLE_EXPECT):
# disable Expect header
curl.setopt(pycurl.HTTPHEADER, ['Expect:'])
# We implement our own redirection-following, because pycurl
# 7.19 doesn't POST after it gets redirected. Ideally we'd be
# using pycurl.POSTREDIR here, but that's in 7.21.
curl.setopt(pycurl.FOLLOWLOCATION, False)
response = StringIO()
curl.setopt(pycurl.WRITEFUNCTION, response.write)
try_counter = connect_retries
while try_counter:
try_counter -= 1
try:
curl.perform()
except pycurl.error as e:
if e.args[0] == 56: # Proxy requires authentication
log_debug(2, e.args[1])
if not (proxy_user and proxy_pass):
raise TransferException("Proxy requires authentication, "
"but reading credentials from "
"%s failed." % YAST_PROXY)
curl.setopt(pycurl.PROXYUSERPWD,
"%s:%s" % (proxy_user, proxy_pass))
elif e.args[0] == 60:
log_error("Peer certificate could not be authenticated "
"with known CA certificates.")
raise TransferException("Peer certificate could not be "
"authenticated with known CA "
"certificates.")
else:
log_error(e.args[1])
raise
status = curl.getinfo(pycurl.HTTP_CODE)
if status == 200 or (URL(url).scheme == "file" and status == 0):
# OK or file
break
elif status in (301, 302): # redirects
url = curl.getinfo(pycurl.REDIRECT_URL)
log_debug(2, "Got redirect to %s" % url)
curl.setopt(pycurl.URL, url)
else:
log_error("Connecting to %s has failed after %s "
"tries with HTTP error code %s." %
(URL(url).getURL(stripPw=True), connect_retries, status))
raise TransferException("Connection failed after %s tries with "
"HTTP error %s." % (connect_retries, status))
# StringIO.write leaves the cursor at the end of the file
response.seek(0)
return response
def fn_transfer(oauth, config, args):
blogEntries = []
directory = args.directory
createUser = args.create_users
print('Loading posts...', end='')
for entry in os.scandir(directory):
if (not entry.is_file()):
continue
if (entry.name == 'authors.yml'):
continue
if (entry.name == 'categories.yml'):
continue
if (not (entry.name.endswith('.yml'))):
continue
with open(entry.path, 'r', encoding='utf-8') as f:
data = yaml.load(f)
blogEntries.append(data)
print('Done.')
print('Extracting post categories...')
# extract categories
categories = set()
for entry in blogEntries:
categories.update(entry['categories'])
# extract authors
with open(directory + '/authors.yml', 'r', encoding='utf-8') as f:
authorMap = yaml.load(f)
authorIds = set()
print('Extracting post authors...')
for entry in blogEntries:
authorIds.add(entry['author_id'])
if (any([e for e in authorIds if e not in authorMap])):
print('Error: Author ID {0:d} unmapped.')
return -2
# trim author map
for k in authorMap:
if (k not in authorIds):
del authorMap[k]
# setup
site = config['url']
site_root = site + '/wp-json{0:s}'
print('Retrieving existing post categories...')
query_params = {
'per_page': str(100),
}
buffer = BytesIO()
c = pycurl.Curl()
url = site_root.format(categories_ep)
c.setopt(c.URL, url + '?' + urlencode(query_params))
c.setopt(c.CAINFO, certifi.where())
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.HTTPHEADER, [oauth.getOAuthHeader('GET', url, query_params)])
c.perform()
# HTTP response code, e.g. 200.
status = c.getinfo(c.RESPONSE_CODE)
if (status != 200):
print('Retrieving existing categories failed...')
return -1
response = json.loads(buffer.getvalue().decode('UTF-8'))
blogCategories = {}
for category in response:
blogCategories[category['name']] = category['id']
category_map = {k: None for k in categories}
for k in category_map:
if (k in blogCategories):
category_map[k] = blogCategories[k]
for k, v in category_map.items():
if (v is None):
print('Creating category {0:s}...'.format(k))
json_data = {
'name': k,
}
post_params = {}
buffer = BytesIO()
c.setopt(c.URL, url)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.HTTPHEADER, [
oauth.getOAuthHeader('POST', url),
'Content-Type: application/json; charset=utf-8'
])
c.setopt(c.POSTFIELDS, json.dumps(json_data))
c.perform()
status = c.getinfo(c.RESPONSE_CODE)
if (status != 201):
print('Creating category failed.')
print(buffer.getvalue().decode('UTF-8'))
return -1
response = json.loads(buffer.getvalue().decode('UTF-8'))
category_map[k] = response['id']
print('Category {0:s} is using ID {1:d}'.format(k, response['id']))
else:
print('Category {0:s} is using ID {1:d}'.format(k, v))
print('Retrieving existing users...')
query_params = {
'per_page': str(100),
}
buffer = BytesIO()
c = pycurl.Curl()
url = site_root.format(users_ep)
c.setopt(c.URL, url + '?' + urlencode(query_params))
c.setopt(c.CAINFO, certifi.where())
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.HTTPHEADER, [oauth.getOAuthHeader('GET', url, query_params)])
c.perform()
# HTTP response code, e.g. 200.
status = c.getinfo(c.RESPONSE_CODE)
if (status != 200):
print('Retrieving existing users failed...')
return -1
response = json.loads(buffer.getvalue().decode('UTF-8'))
blogUsers = {}
for user in response:
blogUsers[user['slug']] = user['id']
if (not (createUser)):
unmappedUsers = [(k, v['slug']) for k, v in authorMap.items()
if v['slug'] not in blogUsers]
if (any(unmappedUsers)):
for id, slug in unmappedUsers:
print('Error: user {0:s} does not exist on blog.'.format(slug))
return -2
else:
for k, v in authorMap.items():
if (v['slug'] in blogUsers):
continue
print('Creating user {0:s}...'.format(v['slug']))
json_data = {
'name': v['name'],
'slug': v['slug'],
'username': v['slug'],
'email': v['slug'] + '@example.com',
'password': '******',
}
post_params = {}
buffer = BytesIO()
c.setopt(c.URL, url)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.HTTPHEADER, [
oauth.getOAuthHeader('POST', url),
'Content-Type: application/json; charset=utf-8'
])
c.setopt(c.POSTFIELDS, json.dumps(json_data).encode('UTF-8'))
c.perform()
status = c.getinfo(c.RESPONSE_CODE)
if (status != 201):
print('Creating user failed.')
print(buffer.getvalue().decode('UTF-8'))
return -1
response = json.loads(buffer.getvalue().decode('UTF-8'))
blogUsers[response['slug']] = response['id']
print('User {0:s} is using ID {1:d}'.format(
response['slug'], response['id']))
for k, v in authorMap.items():
print('User {0:s} is using ID {1:d}'.format(v['slug'],
blogUsers[v['slug']]))
# process posts
url = site_root.format(posts_ep)
for entry in blogEntries:
title = entry['title']
oldAuthorId = entry['author_id']
categories = entry['categories']
comments = [] if ('entries' not in entry['comments']
) else entry['comments']['entries']
content = entry['content']
date = datetime.fromisoformat(entry['date'])
print('Processing \'{0:s}\' with {1:d} comments...'.format(
title, len(comments)))
categoryIds = [category_map[c] for c in categories]
authorId = blogUsers[authorMap[oldAuthorId]['slug']]
# rework content
#
# \r\n<br/>\r\n --> \r\n
# <p>\space*</p> --> kill
# \space+[word]+\r\n[word]+\space+ --> [word]+ [word+]
# \space+-\r\n[word]+ --> \space+- [word]+
content = re.sub(r'\r\n<br/>\r\n', r'\r\n', content)
content = re.sub(r'<p>\s*</p>', r'', content)
content = re.sub(r'\xA0', r'', content) #nbsp;
content = re.sub(r'(\w+)\r\n<a(.*)</a>\r\n(\w+)', r'\1 <a\2</a> \3',
content)
content = re.sub(r'</a>\r\n([.:,])', r'</a>\1', content)
content = re.sub(
r'(\s+)([,.:\w\-\(\)]+)\s*\r\n(?!\d)([.,:\w\-\(\)]+)(\s)',
r'\1\2 \3\4', content)
content = re.sub(
r'(\s+)([,.:\w\-\(\)]+)\r\n\s*(?!\d)([.,:\w\-\(\)]+)(\s)',
r'\1\2 \3\4', content)
content = re.sub(r'(\s+)-\r\n(\s+)([.,\w\-\(\)]+)', r'\1-\2\3',
content)
json_data = {
'date_gmt': date.astimezone(timezone.utc).isoformat(),
'status': 'publish',
'title': title,
'content': content,
'author': str(authorId),
'comment_status': 'closed' if (comments == []) else 'open',
'ping_status': 'closed',
'format': 'standard',
'categories': [str(e) for e in categoryIds]
}
post_params = {}
buffer = BytesIO()
c.setopt(c.URL, url)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.HTTPHEADER, [
oauth.getOAuthHeader('POST', url),
'Content-Type: application/json; charset=utf-8'
])
c.setopt(c.POSTFIELDS, json.dumps(json_data))
c.perform()
status = c.getinfo(c.RESPONSE_CODE)
if (status != 201):
print(' Creating post \'{0:s}\' failed.'.format(title))
print(buffer.getvalue().decode('UTF-8'))
return -1
response = json.loads(buffer.getvalue().decode('UTF-8'))
post_id = response['id']
print(' Created post #{0:d}.'.format(post_id))
if (comments != []):
# create comments
comment_url = site_root.format(comments_ep)
for comment_ix, comment in enumerate(comments):
comment_author = comment['authorName']
comment_date = datetime.fromisoformat(comment['date'])
comment_content = comment['content']
json_data = {
'date_gmt':
comment_date.astimezone(timezone.utc).isoformat(),
'author_name': comment_author,
'author_email': '*****@*****.**',
'content': comment_content,
'post': str(post_id),
'status': 'approve',
}
post_params = {}
buffer = BytesIO()
c.setopt(c.URL, comment_url)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.HTTPHEADER, [
oauth.getOAuthHeader('POST', comment_url),
'Content-Type: application/json; charset=utf-8'
])
c.setopt(c.POSTFIELDS, json.dumps(json_data))
c.perform()
status = c.getinfo(c.RESPONSE_CODE)
if (status != 201):
print(
' Creating comment {0:d} failed.'.format(comment_ix))
print(buffer.getvalue().decode('UTF-8'))
return -1
post_url = '{0:s}/{1:d}'.format(site_root.format(posts_ep),
post_id)
json_data = {
'comment_status': 'closed',
}
post_params = {}
buffer = BytesIO()
c.setopt(c.URL, post_url)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.HTTPHEADER, [
oauth.getOAuthHeader('POST', post_url),
'Content-Type: application/json; charset=utf-8'
])
c.setopt(c.POSTFIELDS, json.dumps(json_data))
c.perform()
status = c.getinfo(c.RESPONSE_CODE)
if (status != 200):
print(' Closing comments for post \'{0:s}\' failed.'.format(
title))
print(buffer.getvalue().decode('UTF-8'))
return -1
return 0
str = BytesIO()
list_head = [
'Accept: */*',
'User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat QBCore/3.43.691.400 QQBrowser/9.0.2524.400'
'X-Requested-With: XMLHttpRequest',
# 'Referer: https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=MzA5OTEzNDkxNw==&scene=124&uin=MTQ0MjA2MDcyMg%3D%3D&key=6d882f3a688ff228c0d62ae7f70433f03aa8ab9870f95f3bb4e7809d55edc7bc2ef60c28ff0c83511f4a1c579a827cd457a640a80b2cb3269b75b30a30de8355ee73b00c6b1a3bdafc67f529a9f6f5bc&devicetype=Windows+7&version=6206021b&lang=zh_CN&a8scene=7&pass_ticket=qkd7lrsCKIkITWLSbXTo22foIjH10JCTcBfvnGIJQfnRHhg7uVXKGSVpJ57SRGcd&winzoom=1',
'Accept-Encoding: gzip, deflate',
'Accept-Language: zh-CN,zh;q=0.8,en-us;q=0.6,en;q=0.5;q=0.4',
'Cookie: rewardsn=; wxtokenkey=777; wxuin=1442060722; devicetype=Windows7; version=6206021b; lang=zh_CN; pass_ticket=qkd7lrsCKIkITWLSbXTo22foIjH10JCTcBfvnGIJQfnRHhg7uVXKGSVpJ57SRGcd; wap_sid2=CLKz0K8FElx0aXFodFBkRmthcUIyak5EeUthclhXWkRyLVlhTW5mX1IxV2JXVjFCLWdPUDI3ZS1JU3NGdjBxaEJyX096U1N1UTJxd1ZEU1BsZk5WYk9lQ1N4RHgyTkFEQUFBfjDag7LdBTgNQAE='
]
postfields = 'r=0.6866572804283351&__biz=MzA5OTEzNDkxNw%3D%3D&appmsg_type=9&mid=2652172289&sn=aacabd39c35587c237366a776b81d259&idx=2&scene=38&title=%25E5%25B8%2588%25E6%2581%25A9%25E9%259A%25BE%25E5%25BF%2598%25EF%25BC%258C%25E6%2584%259F%25E8%25B0%25A2%25E6%258C%2587%25E5%25BC%2595%25E5%2589%258D%25E8%25A1%258C%25E7%259A%2584%25E6%2582%25A8&ct=1536591292&abtest_cookie=&devicetype=Windows%207&version=6206021b&is_need_ticket=0&is_need_ad=0&comment_id=452968419168681984&is_need_reward=0&both_ad=0&reward_uin_count=0&send_time=&msg_daily_idx=1&is_original=0&is_only_read=1&req_id=2717AUvZQVDkCI022y0tkIEe&pass_ticket=qkd7lrsCKIkITWLSbXTo22foIjH10JCTcBfvnGIJQfnRHhg7uVXKGSVpJ57SRGcd&is_temp_url=0&item_show_type=undefined&tmp_version=1'
# url = 'https://mp.weixin.qq.com/mp/appmsg_comment?action=getcomment&scene=0&__biz=MzA5OTEzNDkxNw==&appmsgid=2652172289&idx=2&comment_id=452968419168681984&offset=0&limit=100&uin=MTQ0MjA2MDcyMg%253D%253D&key=bd90134f48e7cdcf3a4948cd8783c250b255ac197e735b20573c7693526260ebd913defbbca92d2c1c6ead07582d88f091f7c10db70c1ac3396035bddf05fbec290676aa37117dac8c86c4e7cf4cdcc5&pass_ticket=qkd7lrsCKIkITWLSbXTo22foIjH10JCTcBfvnGIJQfnRHhg7uVXKGSVpJ57SRGcd&wxtoken=777&devicetype=Windows%26nbsp%3B7&clientversion=6206021b&appmsg_token=976_YVZCIiDTrLQkl%252BOlsEV_nnCxjKLXIicmk6nweJkSIyYVRfHOqang8yo3RgNTU7IHgloLFpXdEHOyrEPs&x5=0&f=json'
cobj = pycurl.Curl()
url = [
'https://www.baidu.com/', 'http://t.cn/aKln8T',
'https://blog.csdn.net/cityzenoldwang/article/details/78621337'
]
for i in url:
cobj.setopt(pycurl.URL, i)
cobj.setopt(pycurl.CAINFO, certifi.where())
cobj.setopt(pycurl.CONNECTTIMEOUT, 10)
cobj.setopt(pycurl.WRITEFUNCTION, str.write)
# cobj.setopt(pycurl.VERBOSE, 1)
# cobj.setopt(pycurl.POSTFIELDS,postfields)
# cobj.setopt(pycurl.NOPROGRESS,0)
# cobj.setopt(pycurl.HTTPHEADER, list_head)
def fn_register(oauth, config):
if ('oauthCallback' not in config or config['oauthCallback'] != 'oob'):
print('oauthCallback missing from config or not \'oob\'...')
return -1
site = config['url']
oauth1_request_url = site + '/oauth1/request'
oauth1_authorize_url = site + '/oauth1/authorize'
oauth1_access_url = site + '/oauth1/access'
c = pycurl.Curl()
c.setopt(c.CAINFO, certifi.where())
#c.setopt(c.VERBOSE, True)
buffer = BytesIO()
post_params = {}
c.setopt(c.POST, True)
c.setopt(c.URL, oauth1_request_url)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.HTTPHEADER, [
oauth.getOAuthHeader('POST', oauth1_request_url, post_params,
{'oauth_callback': config['oauthCallback']})
])
c.setopt(c.POSTFIELDS, '')
c.perform()
status = c.getinfo(c.RESPONSE_CODE)
if (status != 200):
print('Requesting Authorization failed...')
return -1
response = buffer.getvalue().decode('utf-8')
params = parse_qs(response)
if (not (all(e in params for e in ['oauth_token', 'oauth_token_secret']))):
print(
'Authorization Response did not contain required token and secret.'
)
return -2
print('Please visit the following URL:\n'
'\n' + oauth1_authorize_url + '?' + response)
oauthVerifier = input('Please insert verifier (or nothing to cancel):')
if (oauthVerifier == ''):
print('Authorization aborted...')
return -3
print('Verifier: ' + oauthVerifier)
oauth.updateOAuthToken(params['oauth_token'][-1],
params['oauth_token_secret'][-1])
post_params = {}
add_oauth_param = {
'oauth_verifier': oauthVerifier,
'oauth_callback': config['oauthCallback']
}
buffer = BytesIO()
c.setopt(c.POST, True)
c.setopt(c.URL, oauth1_access_url)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.HTTPHEADER, [
oauth.getOAuthHeader('POST', oauth1_access_url, post_params,
add_oauth_param)
])
c.setopt(c.POSTFIELDS, '')
c.perform()
status = c.getinfo(c.RESPONSE_CODE)
if (status != 200):
print('Accessing OAuth 1.0a failed...')
return -4
response = buffer.getvalue().decode('utf-8')
params = parse_qs(response)
if (not (all(e in params for e in ['oauth_token', 'oauth_token_secret']))):
print(
'Authorization Response did not contain required token and secret.'
)
return -5
config['oauthToken'] = params['oauth_token'][-1]
config['oauthTokenSecret'] = params['oauth_token_secret'][-1]
print('Updated Config:')
print('-' * 72)
print(yaml.dump(config, default_flow_style=False))
print('-' * 72)
return 0
payload['topic'] = "home/downstairs/playroom/sensor/motion"
payload['retain'] = "True"
data_payload = {}
data_payload['status'] = "MOTION"
data_payload['cam_id'] = ''
data_payload['width'] = ''
data_payload['height'] = ''
data_payload['x'] = ''
data_payload['y'] = ''
data_payload['event_id'] = ''
data_payload['pixels'] = ''
data_payload['noise'] = ''
for opt, arg in opts:
opt = opt.replace("-", "")
data_payload[opt] = arg
payload['payload'] = json.dumps(data_payload)
json_string = json.dumps(payload)
request = pycurl.Curl()
request.setopt(pycurl.WRITEFUNCTION, lambda x: None)
request.setopt(request.URL, base_url)
request.setopt(request.HTTPHEADER, ["Content-Type: application/json"])
request.setopt(request.POSTFIELDS, json_string)
request.perform()
request.close()
# 利用这些信息可以定位服务响应慢的具体细节。
# pip install pycurl
import pycurl
# 检测安装结果
print pycurl.version
# PycURL/7.43.0.1 libcurl/7.57.0 OpenSSL/1.1.0g zlib/1.2.11 c-ares/1.13.0 libssh2/1.8.0
# 模块常用方法说明：
# pycurl.Curl()类：实现创建一个 libcurl 包的Curl 句柄对象，无参数。
# 1.close()方法：对应 libcurl 包中的 curl_easy_cleanup 方法，无参数，实现关闭，回收Curl 对象。
# 2.perform()方法：对应 libcurl 包中的 curl_easy_perform 方法，无参数，实现Curl 对象请求的提交。
# 3.setopt(option, value)方法：对应libcurl 包中的 curl_easy_setopt 方法，参数 iption 是通过
# libcurl 的常量来指定的，参数 value 的值会依赖option，可以是一个字符串、整型、长整型、文件对象、列表或函数。
c = pycurl.Curl() # 创建一个 curl 对象
c.setopt(pycurl.CONNECTTIMEOUT, 5) # 连接的等待时间，设置为0 则不等待
c.setopt(pycurl.TIMEOUT, 5) # 请求超时时间
c.setopt(pycurl.NOPROGRESS, 0) # 是否屏蔽下载进度条， 非 0 则屏蔽
c.setopt(pycurl.MAXREDIRS, 5) # 指定 HTTP 重定向的最大数
c.setopt(pycurl.FORBID_REUSE, 1) # 完成交互后强制断开连接，不重用
c.setopt(pycurl.FRESH_CONNECT, 1) # 强制获取新的连接，即替代缓存中的连接
c.setopt(pycurl.DNS_CACHE_TIMEOUT, 60) # 设置保存DNS 信息的时间，默认为120秒
c.setopt(pycurl.URL, "http://www.baidu.com") # 指定请求的URL
c.setopt(pycurl.USERAGENT, "Mozilla/5.2 (compatible; MSIE 6.0; Windows NT 5.1;\
SV1; .NET CLR 1.14322; .NET CLR 2.0.50324)") # 配置请求HTTP头的User-Agent
c.setopt(pycurl.HEADERFUNCTION,
getheader) # 将返回的 HTTP HEADER 定向到回调函数 getheader
c.setopt(pycurl.WRITEFUNCTION, getbody) # 将返回的内容定向到回调函数 getbody
c.setopt(pycurl.WRITEHEADER, fileobj) # 将返回的 HTTP HEADER 定向到fileobj 文件对象
c.setopt(pycurl.WRITEDATA, fileobj) # 将返回的 HTML 内容定向到fileobj 文件对象
def Test1():
crl = pycurl.Curl()
crl.setopt(pycurl.VERBOSE, 1)
crl.setopt(crl.URL, "http://graph.cloudiya.com/LICENSE.txt")
import couchdb
import pycurl
import json
user = "******"
password = "******"
db_ip = '127.0.0.1'
couchserver = couchdb.Server('http://%s:%s@%s:5984/' % (user, password, db_ip))
aurindb = "low_income_households"
aurindb2 = "population"
dbname = "travis_data"
# add the Aurin db low_income_households if not exists
if aurindb not in couchserver:
# create db
crl = pycurl.Curl()
crl.setopt(
crl.URL, 'http://%s:%s@%s:5984/%s?partitioned=true' %
(user, password, db_ip, aurindb))
crl.setopt(crl.UPLOAD, 1)
crl.perform()
crl.close()
# add data
crl = pycurl.Curl()
crl.setopt(
crl.URL, 'http://%s:%s@%s:5984/%s/_bulk_docs' %
(user, password, db_ip, aurindb))
crl.setopt(pycurl.HTTPHEADER, ["Content-Type: application/json"])
f = open('./data/lowIncomeProcessed.json', 'r')
data = f.read()
crl.setopt(pycurl.POST, 1)
def http_download(url,
port=0,
method='GET',
params={},
headers={},
timeout=15,
proxy=None):
r"""
Helper function to download files from website
This function will apply proxy settings and deal redirections automatically.
To apply proxy settings, pass an ProxySettings object as the `proxy` parameter.
If `'User-Agent'` is not set in `headers`, it will be set to `'OSD Lyrics'`.
Arguments:
- `url`: The url of the content
- `port`: (optional) The port.
- `method`: (optional) The HTTP method to download contents. Available values
are `'POST'` or `'GET'`. The default value is `'GET'`.
- `params`: (optional) The parameters of the request. It is a dict. If `method`
is `'GET'`, `params` will be encoded and append to the url as the
param part. If `method` is `'POST'`, `params` will be added to
request headers as post data.
- `headers`: (optional) A dict of HTTP headers.
- `proxy`: (optional) A ProxySettings object to sepcify the proxy to use.
>>> code, content = http_download('http://www.python.org/')
>>> code
200
>>> content.find('Python') >= 0
True
"""
c = pycurl.Curl()
buf = StringIO.StringIO()
c.setopt(pycurl.NOSIGNAL, 1)
c.setopt(pycurl.DNS_USE_GLOBAL_CACHE, 0)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.MAXREDIRS, 5)
c.setopt(pycurl.WRITEFUNCTION, buf.write)
if method == 'GET' and len(params) > 0:
params = urllib.urlencode(params)
url = url + ('/' if '/' not in url else '') + ('?' if '?' not in url
else '&') + params
elif method == 'POST':
c.setopt(pycurl.POST, 1)
if len(params) > 0:
c.setopt(pycurl.POSTFIELDS, params) #Someone had forgot an 'S'
c.setopt(pycurl.POSTFIELDSIZE, len(params))
url = ensure_utf8(url)
c.setopt(pycurl.URL, url)
if port > 0 and port < 65536:
c.setopt(pycurl.PORT, port)
real_headers = {'User-Agent': 'OSD Lyrics'}
real_headers.update(headers)
curl_headers = ['%s:%s' % (k, v) for k, v in real_headers.items()]
c.setopt(pycurl.HTTPHEADER, curl_headers)
if proxy is not None and proxy.protocol != 'no':
if proxy.username != '' and proxy.username is not None:
proxyvalue = '%s://%s:%s@%s:%d' % (proxy.protocol, proxy.username,
proxy.password, proxy.host,
proxy.port)
else:
proxyvalue = '%s://%s:%d' % (proxy.protocol, proxy.host,
proxy.port)
c.setopt(pycurl.PROXY, proxyvalue)
else:
c.setopt(pycurl.PROXY, '')
c.perform()
return c.getinfo(pycurl.HTTP_CODE), buf.getvalue()
def main(order, downloadLimit, verbose):
if verbose:
print("Connecting to order " + order)
# Build the ftp host with the order id
host = "ftp://ladsweb.nascom.nasa.gov/orders/" + order + "/"
# Initiate curl
c = pycurl.Curl()
c.setopt(pycurl.URL, host)
# String output buffer for curl return
output = BytesIO()
c.setopt(pycurl.WRITEFUNCTION, output.write)
# Execute curl and get the order from the output buffer
c.perform()
order = output.getvalue().decode('UTF-8').split()
dlCount = 0
# Let's get a list of both the HDF03s and HDF02s
HDF03 = [hdf for hdf in order if ".hdf" in hdf and "D03" in hdf]
HDF02 = [hdf for hdf in order if ".hdf" in hdf and "D02" in hdf]
# Download all HDF02s with the corresponding HDF03s if they exist
for hdf02 in HDF02:
# Parse the HDF02 in order to get the corresponding HDF03
filSplt = hdf02.split('.')
datTim = filSplt[1].replace('A', '') + filSplt[2]
t = datetime.datetime.strptime(datTim, "%Y%j%H%M")
julianDay = str(t.timetuple().tm_yday)
jZeros = 3 - len(julianDay)
julianDay = '0' * jZeros + julianDay
yr = str(t.year)
hr = str(t.hour)
hrZeros = 2 - len(hr)
hr = '0' * hrZeros + hr
mint = str(t.minute)
mintZeros = 2 - len(mint)
mint = '0' * mintZeros + mint
datNam = yr + julianDay + '.' + hr + mint
# Check to see if the HDF03 exists in the HDF03 list
for filNamCandidate in HDF03:
if datNam in filNamCandidate:
hdf03 = filNamCandidate
break
# Both a HDF02 and HDF03 have been found
if hdf03:
if os.path.exists(hdf02) and int(
order[order.index(hdf02) - 4]) == os.path.getsize(hdf02):
if verbose:
print("Skipping download of " + hdf02)
else:
if verbose:
print("Attempting download of " + hdf02)
fp = open(os.path.join('.', hdf02), "wb")
curl = pycurl.Curl()
curl.setopt(pycurl.URL, host + hdf02)
curl.setopt(pycurl.WRITEDATA, fp)
curl.perform()
curl.close()
fp.close()
if verbose:
print("Successfully downloaded " + hdf02)
if os.path.exists(hdf03) and int(
order[order.index(hdf03) - 4]) == os.path.getsize(hdf03):
if verbose:
print("Skipping download of " + hdf03)
else:
if verbose:
print("Attempting download of " + hdf03)
fp = open(os.path.join('.', hdf03), "wb")
curl = pycurl.Curl()
curl.setopt(pycurl.URL, host + hdf03)
curl.setopt(pycurl.WRITEDATA, fp)
curl.perform()
curl.close()
fp.close()
if verbose:
print("Successfully downloaded " + hdf03)
dlCount += 1
if downloadLimit == dlCount:
if verbose:
print("HDF download limit reached")
break
if verbose:
print("FTP download of order successful")
def setUp(self):
self.curl = pycurl.Curl()
def configure_curl(self,
timeout=DEFAULT_TIMEOUT,
context=None,
curl_handle=None):
""" Create and mostly configure a curl object for test, reusing existing if possible """
if curl_handle:
curl = curl_handle
else:
curl = pycurl.Curl()
# curl.setopt(pycurl.VERBOSE, 1) # Debugging convenience
curl.setopt(curl.URL, str(self.url))
curl.setopt(curl.TIMEOUT, timeout)
bod = self.body
# Set read function for post/put bodies
if self.method == u'POST' or self.method == u'PUT':
if bod and len(bod) > 0:
curl.setopt(curl.READFUNCTION, StringIO(bod).read)
#else:
# curl.setopt(curl.READFUNCTION, lambda x: None)
if self.auth_username and self.auth_password:
curl.setopt(pycurl.USERPWD,
'%s:%s' % (self.auth_username, self.auth_password))
if self.auth_type:
curl.setopt(pycurl.HTTPAUTH, self.auth_type)
if self.method == u'POST':
curl.setopt(HTTP_METHODS[u'POST'], 1)
# Required for some servers
if bod is not None:
curl.setopt(pycurl.POSTFIELDSIZE, len(bod))
else:
curl.setopt(pycurl.POSTFIELDSIZE, 0)
elif self.method == u'PUT':
curl.setopt(HTTP_METHODS[u'PUT'], 1)
# Required for some servers
if bod is not None:
curl.setopt(pycurl.INFILESIZE, len(bod))
else:
curl.setopt(pycurl.INFILESIZE, 0)
elif self.method == u'DELETE':
curl.setopt(curl.CUSTOMREQUEST, 'DELETE')
head = self.get_headers(context=context)
if head: #Convert headers dictionary to list of header entries, tested and working
headers = [
str(headername) + ':' + str(headervalue)
for headername, headervalue in head.items()
]
else:
headers = list()
headers.append(
"Expect:"
) # Fix for expecting 100-continue from server, which not all servers will send!
headers.append("Connection: close")
curl.setopt(curl.HTTPHEADER, headers)
return curl
def fetch(self, url, body=None, headers=None,cargs=None):
stop = int(time.time()) + self.ALLOWED_TIME
off = self.ALLOWED_TIME
if headers is None:
headers = {}
headers.setdefault('User-Agent',
"%s %s" % (USER_AGENT, pycurl.version,))
header_list = []
if headers is not None:
for header_name, header_value in headers.iteritems():
header_list.append('%s: %s' % (header_name, header_value))
c = pycurl.Curl()
try:
c.setopt(pycurl.NOSIGNAL, 1)
if header_list:
c.setopt(pycurl.HTTPHEADER, header_list)
if cargs is not None:
for key,val in cargs.iteritems():
c.setopt(getattr(pycurl,key),val)
# Presence of a body indicates that we should do a POST
if body is not None:
c.setopt(pycurl.POST, 1)
c.setopt(pycurl.POSTFIELDS, body)
while off > 0:
if not self._checkURL(url):
raise HTTPError("Fetching URL not allowed: %r" % (url,))
data = cStringIO.StringIO()
def write_data(chunk):
if data.tell() > 1024*MAX_RESPONSE_KB:
return 0
else:
return data.write(chunk)
response_header_data = cStringIO.StringIO()
c.setopt(pycurl.WRITEFUNCTION, write_data)
c.setopt(pycurl.HEADERFUNCTION, response_header_data.write)
c.setopt(pycurl.TIMEOUT, off)
c.setopt(pycurl.URL, openid.urinorm.urinorm(url))
c.perform()
response_headers = self._parseHeaders(response_header_data)
code = c.getinfo(pycurl.RESPONSE_CODE)
if code in [301, 302, 303, 307]:
url = response_headers.get('location')
if url is None:
raise HTTPError(
'Redirect (%s) returned without a location' % code)
# Redirects are always GETs
c.setopt(pycurl.POST, 0)
# There is no way to reset POSTFIELDS to empty and
# reuse the connection, but we only use it once.
else:
resp = HTTPResponse()
resp.headers = response_headers
resp.status = code
resp.final_url = url
resp.body = data.getvalue()
return resp
off = stop - int(time.time())
raise HTTPError("Timed out fetching: %r" % (url,))
finally:
c.close()
def single_file_upload(file):
""" This method presumes you are using ResourceSpace. If you are not, you will need to
modify the method to properly make the API call and handle the response. """
# create new upload batch
current_time = datetime.strftime(datetime.now(), '%Y-%m-%d_%H-%M-%S')
upload_batch = UploadBatch(name="Single File Upload " + current_time,
data_upload=0)
upload_batch.save()
name_only, ext = splitext(file)
upload_file_loc = settings.FILE_UPLOAD_FOLDER + '/' + file
curl = pycurl.Curl()
storage = StringIO()
data = [
('userfile', (pycurl.FORM_FILE, upload_file_loc)),
('key', settings.FILE_UPLOAD_API_KEY),
('field8', "File Upload " + current_time
) #this is very specific to UrOnline installation
]
curl.setopt(pycurl.URL, settings.FILE_UPLOAD_API_URI)
curl.setopt(pycurl.HTTPPOST, data)
curl.setopt(pycurl.WRITEFUNCTION, storage.write)
curl.setopt(pycurl.USERAGENT, "API Client")
curl.perform()
curl.close()
content = storage.getvalue()
json_content = json.loads(content)
new_rsid = int(json_content['resource'][0]['ref'])
# create new file in main db
new_file = File(id=new_rsid, upload_batch=upload_batch, filetype=ext[1:])
new_file.save()
# create property for file name
fn_prop_test = ResultProperty.objects.filter(display_field='file_name')
# create a file name result property if one isn't designated
if not (fn_prop_test and fn_prop_test[0].field_type):
dp = DescriptiveProperty(property='Filename',
last_mod_by=User.objects.get(pk=1),
primary_type='MF',
visible=0)
dp.save()
fn_prop_test.field_type = dp
fn_prop_test.save()
else:
dp = fn_prop_test[0].field_type
new_fp = FileProperty(file=new_file,
property=dp,
property_value=file,
last_mod_by=User.objects.get(pk=1),
upload_batch=upload_batch)
new_fp.save()
# delete file from temp storage now that its uploaded
remove(upload_file_loc)
return new_rsid
def test_fail(self):
curl = pycurl.Curl()
self.assertEqual(1, curl.SEEKFUNC_FAIL)
curl.close()
'''So we are trying to collect data from coursera using the catalog api.'''
from pprint import pprint
import pycurl
from StringIO import StringIO
import ujson
basic_url = 'https://api.coursera.org/api/courses.v1'
start = 0
limit = 100
fields = 'language,shortDescription'
# url = basic_url + '?' + 'start=' + str(start) + '&' + 'limit=' + str(limit) + '&' + 'fields=' + fields
# url = "https://api.coursera.org/api/courses.v1?ids=Z3yHdBVBEeWvmQrN_lODCw,v1-3,Gtv4Xb1-EeS-ViIACwYKVQ&fields=language,shortDescription,previewLink"
url = "https://api.coursera.org/api/courses.v1?ids=v1-228,v1-3,69Bku0KoEeWZtA4u62x6lQ,0HiU7Oe4EeWTAQ4yevf_oQ,5zjIsJq-EeW_wArffOXkOw,v9CQdBkhEeWjrA6seF25aw,QgmoVdT2EeSlhSIACx2EBw,QTTRdG3vEeWG0w42Sx-Gkw,v1-1355,5uXCfFu2EeSU0SIACxCMgg,Rr2thkIKEeWZtA4u62x6lQ,rajsT7UJEeWl_hJObLDVwQ,lfKT5sS3EeWhPQ55lNYVVQ,v1-1250,POZJ3uOtEeSoXCIACw4Gzg,ugSnwH9hEeSiIiIAC3lQMQ,rc5KG0aUEeWG1w6arGoEIQ,gpAI9GK4EeWFkQ7sUCFGVQ,v1-640,Kzg9QkDxEeWZtA4u62x6lQ,tEqImn2kEeWb-BLhFdaGww&fields=previewLink,language,shortDescription"
print url
raw_input('url dekho')
bufferr = StringIO()
curl_instance = pycurl.Curl()
curl_instance.setopt(curl_instance.URL, url)
curl_instance.setopt(curl_instance.WRITEDATA, bufferr)
curl_instance.setopt(curl_instance.VERBOSE, True)
curl_instance.perform()
curl_instance.close()
body = bufferr.getvalue()
print body
body_json = ujson.loads(body)
pprint(body_json)
start = body_json['paging']['next']
def run_benchmark(benchmark, test_config=TestConfig(), context=None):
""" Perform a benchmark, (re)using a given, configured CURL call to do so
The actual analysis of metrics is performed separately, to allow for testing
"""
# Context handling
my_context = context
if my_context is None:
my_context = Context()
warmup_runs = benchmark.warmup_runs
benchmark_runs = benchmark.benchmark_runs
message = '' # Message is name of benchmark... print it?
if (benchmark_runs <= 0):
raise Exception("Invalid number of benchmark runs, must be > 0 :" +
benchmark_runs)
result = TestResponse()
# TODO create and use a curl-returning configuration function
# TODO create and use a post-benchmark cleanup function
# They should use is_dynamic/is_context_modifier to determine if they need to
# worry about context and re-reading/retemplating and only do it if needed
# - Also, they will need to be smart enough to handle extraction functions
# For performance reasons, we don't want to re-run templating/extraction if
# we do not need to, and do not want to save request bodies.
# Initialize variables to store output
output = BenchmarkResult()
output.name = benchmark.name
output.group = benchmark.group
metricnames = list(benchmark.metrics)
metricvalues = [
METRICS[name] for name in metricnames
] # Metric variable for curl, to avoid hash lookup for every metric name
results = [list() for x in xrange(0, len(metricnames))
] # Initialize arrays to store results for each metric
curl = pycurl.Curl()
# Benchmark warm-up to allow for caching, JIT compiling, on client
logger.info('Warmup: ' + message + ' started')
for x in xrange(0, warmup_runs):
benchmark.update_context_before(my_context)
templated = benchmark.realize(my_context)
curl = templated.configure_curl(timeout=test_config.timeout,
context=my_context,
curl_handle=curl)
curl.setopt(
pycurl.WRITEFUNCTION,
lambda x: None) # Do not store actual response body at all.
curl.perform()
logger.info('Warmup: ' + message + ' finished')
logger.info('Benchmark: ' + message + ' starting')
for x in xrange(0, benchmark_runs): # Run the actual benchmarks
# Setup benchmark
benchmark.update_context_before(my_context)
templated = benchmark.realize(my_context)
curl = templated.configure_curl(timeout=test_config.timeout,
context=my_context,
curl_handle=curl)
curl.setopt(
pycurl.WRITEFUNCTION,
lambda x: None) # Do not store actual response body at all.
try: # Run the curl call, if it errors, then add to failure counts for benchmark
curl.perform()
except Exception:
output.failures = output.failures + 1
curl.close()
curl = pycurl.Curl()
continue # Skip metrics collection
# Get all metrics values for this run, and store to metric lists
for i in xrange(0, len(metricnames)):
results[i].append(curl.getinfo(metricvalues[i]))
curl.close()
logger.info('Benchmark: ' + message + ' ending')
temp_results = dict()
for i in xrange(0, len(metricnames)):
temp_results[metricnames[i]] = results[i]
output.results = temp_results
return analyze_benchmark_results(output, benchmark)
def get_metrics_for_hub_url(self, url):
#FNULL = open(os.devnull, 'w')
#expected_status_code = "200"
#return subprocess.call("curl 'https://" + url + "' -H 'Connection: keep-alive'
# -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'Upgrade-Insecure-Requests: 1'
# -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
# -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
# -H 'Accept-Encoding: gzip, deflate, br' -H 'Accept-Language: en-US,en;q=0.9'
# --compressed --insecure -I -s | grep 'HTTP/1.1[ ]" +
# expected_status_code + "'", shell=True, stdout=FNULL, stderr=subprocess.STDOUT) == 0;
buffer = StringIO()
c = pycurl.Curl()
c.setopt(c.URL, 'https://' + url)
c.setopt(c.WRITEDATA, buffer)
c.setopt(c.CAINFO, certifi.where())
if self._insecure:
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
#if self.verbose:
# c.setopt(c.VERBOSE, True)
c.setopt(pycurl.USERAGENT,'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36')
c.setopt(pycurl.ENCODING, 'gzip, deflate, br')
c.setopt(pycurl.HTTPHEADER,
['Connection: keep-alive',
'Pragma: no-cache',
'Cache-Control: no-cache',
'Upgrade-Insecure-Requests: 1',
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
'Accept-Language: en-US,en;q=0.9'
])
c.perform()
record = {}
#record['ACTIVESOCKET'] = c.getinfo(pycurl.ACTIVESOCKET)
#record['APPCONNECT_TIME_T'] = c.getinfo(pycurl.APPCONNECT_TIME_T)
#record['CERTINFO'] = c.getinfo(pycurl.CERTINFO)
#record['CONDITION_UNMET'] = c.getinfo(pycurl.CONDITION_UNMET)
#record['CONNECT_TIME_T'] = c.getinfo(pycurl.CONNECT_TIME_T)
#record['CONTENT_LENGTH_DOWNLOAD_T'] = c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD_T)
#record['CONTENT_LENGTH_UPLOAD_T'] = c.getinfo(pycurl.CONTENT_LENGTH_UPLOAD_T)
#record['COOKIELIST'] = c.getinfo(pycurl.COOKIELIST)
#record['FILETIME'] = c.getinfo(pycurl.FILETIME)
#record['FILETIME_T'] = c.getinfo(pycurl.FILETIME_T)
#record['HTTP_VERSION'] = c.getinfo(pycurl.HTTP_VERSION)
#record['NAMELOOKUP_TIME_T'] = c.getinfo(pycurl.NAMELOOKUP_TIME_T)
#record['PRETRANSFER_TIME_T'] = c.getinfo(pycurl.PRETRANSFER_TIME_T)
#record['PRIVATE'] = c.getinfo(pycurl.PRIVATE)
#record['PROTOCOL'] = c.getinfo(pycurl.PROTOCOL)
#record['PROXY_SSL_VERIFYRESULT'] = c.getinfo(pycurl.PROXY_SSL_VERIFYRESULT)
#record['REDIRECT_TIME_T'] = c.getinfo(pycurl.REDIRECT_TIME_T)
#record['RTSP_CLIENT_CSEQ'] = c.getinfo(pycurl.RTSP_CLIENT_CSEQ)
#record['RTSP_CSEQ_RECV'] = c.getinfo(pycurl.RTSP_CSEQ_RECV)
#record['RTSP_SERVER_CSEQ'] = c.getinfo(pycurl.RTSP_SERVER_CSEQ)
#record['RTSP_SESSION_ID'] = c.getinfo(pycurl.RTSP_SESSION_ID)
#record['SCHEME'] = c.getinfo(pycurl.SCHEME)
#record['SIZE_DOWNLOAD_T'] = c.getinfo(pycurl.SIZE_DOWNLOAD_T)
#record['SIZE_UPLOAD_T'] = c.getinfo(pycurl.SIZE_UPLOAD_T)
#record['SPEED_DOWNLOAD_T'] = c.getinfo(pycurl.SPEED_DOWNLOAD_T)
#record['SPEED_UPLOAD_T'] = c.getinfo(pycurl.SPEED_UPLOAD_T)
#record['STARTTRANSFER_TIME_T'] = c.getinfo(pycurl.STARTTRANSFER_TIME_T)
#record['TLS_SESSION'] = c.getinfo(pycurl.TLS_SESSION)
#record['TLS_SSL_PTR'] = c.getinfo(pycurl.TLS_SSL_PTR)
#record['TOTAL_TIME_T'] = c.getinfo(pycurl.TOTAL_TIME_T)
record['APPCONNECT_TIME'] = c.getinfo(pycurl.APPCONNECT_TIME)
record['CONNECT_TIME'] = c.getinfo(pycurl.CONNECT_TIME)
record['CONTENT_LENGTH_DOWNLOAD'] = c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)
record['CONTENT_LENGTH_UPLOAD'] = c.getinfo(pycurl.CONTENT_LENGTH_UPLOAD)
record['CONTENT_TYPE'] = c.getinfo(pycurl.CONTENT_TYPE)
record['EFFECTIVE_URL'] = c.getinfo(pycurl.EFFECTIVE_URL)
record['FTP_ENTRY_PATH'] = c.getinfo(pycurl.FTP_ENTRY_PATH)
record['HEADER_SIZE'] = c.getinfo(pycurl.HEADER_SIZE)
record['HTTPAUTH_AVAIL'] = c.getinfo(pycurl.HTTPAUTH_AVAIL)
record['HTTP_CONNECTCODE'] = c.getinfo(pycurl.HTTP_CONNECTCODE)
record['LASTSOCKET'] = c.getinfo(pycurl.LASTSOCKET)
record['LOCAL_IP'] = c.getinfo(pycurl.LOCAL_IP)
record['LOCAL_PORT'] = c.getinfo(pycurl.LOCAL_PORT)
record['NAMELOOKUP_TIME'] = c.getinfo(pycurl.NAMELOOKUP_TIME)
record['NUM_CONNECTS'] = c.getinfo(pycurl.NUM_CONNECTS)
record['OS_ERRNO'] = c.getinfo(pycurl.OS_ERRNO)
record['PRETRANSFER_TIME'] = c.getinfo(pycurl.PRETRANSFER_TIME)
record['PRIMARY_IP'] = c.getinfo(pycurl.PRIMARY_IP)
record['PRIMARY_PORT'] = c.getinfo(pycurl.PRIMARY_PORT)
record['PROXYAUTH_AVAIL'] = c.getinfo(pycurl.PROXYAUTH_AVAIL)
record['REDIRECT_COUNT'] = c.getinfo(pycurl.REDIRECT_COUNT)
record['REDIRECT_TIME'] = c.getinfo(pycurl.REDIRECT_TIME)
record['REDIRECT_URL'] = c.getinfo(pycurl.REDIRECT_URL)
record['REQUEST_SIZE'] = c.getinfo(pycurl.REQUEST_SIZE)
record['RESPONSE_CODE'] = c.getinfo(pycurl.RESPONSE_CODE)
record['SIZE_DOWNLOAD'] = c.getinfo(pycurl.SIZE_DOWNLOAD)
record['SIZE_UPLOAD'] = c.getinfo(pycurl.SIZE_UPLOAD)
record['SPEED_DOWNLOAD'] = c.getinfo(pycurl.SPEED_DOWNLOAD)
record['SPEED_UPLOAD'] = c.getinfo(pycurl.SPEED_UPLOAD)
record['SSL_ENGINES'] = c.getinfo(pycurl.SSL_ENGINES)
record['SSL_VERIFYRESULT'] = c.getinfo(pycurl.SSL_VERIFYRESULT)
record['STARTTRANSFER_TIME'] = c.getinfo(pycurl.STARTTRANSFER_TIME)
record['TOTAL_TIME'] = c.getinfo(pycurl.TOTAL_TIME)
c.close()
return record
def login():
try:
os.remove("instanax.txt")
except:
pass
# getting username and password
username = arguments['<user>']
password = getpass.getpass()
buf = cStringIO.StringIO()
c = pycurl.Curl()
c.setopt(pycurl.URL, "http://web.stagram.com")
c.setopt(pycurl.COOKIEFILE, "instanax.txt")
c.setopt(pycurl.COOKIEJAR, "instanax.txt")
c.setopt(pycurl.WRITEFUNCTION, buf.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.ENCODING, "")
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
useragent = UA()
c.setopt(pycurl.USERAGENT, useragent)
c.perform()
curlData = buf.getvalue()
buf.close()
clientid = re.findall(ur"href=\"https:\/\/api.instagram.com\/oauth\/authorize\/\?client_id=([a-z0-9]*)&redirect_uri=http:\/\/web.stagram.com\/&response_type=code&scope=likes\+comments\+relationships\">LOG IN",curlData)
instagramlink = re.findall(ur"href=\"([^\"]*)\">LOG IN",curlData)
buf = cStringIO.StringIO()
c = pycurl.Curl()
c.setopt(pycurl.URL, instagramlink[0])
c.setopt(pycurl.COOKIEFILE, "instanax.txt")
c.setopt(pycurl.COOKIEJAR, "instanax.txt")
c.setopt(pycurl.WRITEFUNCTION, buf.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.ENCODING, "")
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
useragent = UA()
c.setopt(pycurl.USERAGENT, useragent)
c.perform()
curlData = buf.getvalue()
buf.close()
postaction = re.findall(ur"action=\"([^\"]*)\"",curlData)
csrfmiddlewaretoken = re.findall(ur"name=\"csrfmiddlewaretoken\" value=\"([^\"]*)\"",curlData)
postdata = 'csrfmiddlewaretoken='+csrfmiddlewaretoken[0]+'&username='******'&password='+password
buf = cStringIO.StringIO()
c = pycurl.Curl()
c.setopt(pycurl.URL, "https://instagram.com"+postaction[0])
c.setopt(pycurl.COOKIEFILE, "instanax.txt")
c.setopt(pycurl.COOKIEJAR, "instanax.txt")
c.setopt(pycurl.WRITEFUNCTION, buf.write)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.ENCODING, "")
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
c.setopt(pycurl.REFERER, "https://instagram.com/accounts/login/?next=/oauth/authorize/%3Fclient_id%3D"+clientid[0]+"%26redirect_uri%3Dhttp%3A//web.stagram.com/%26response_type%3Dcode%26scope%3Dlikes%2Bcomments%2Brelationships")
useragent = UA()
c.setopt(pycurl.USERAGENT, useragent)
c.setopt(pycurl.POST, 1)
c.setopt(pycurl.POSTFIELDS, postdata)
c.setopt(pycurl.POSTFIELDSIZE, len(postdata))
#c.setopt(pycurl.VERBOSE, True)
c.perform()
curlData = buf.getvalue()
buf.close()
def download_data(self, url):
transfer_stats = []
gwdatafind_server = url[(url.find("://") + 3):url.find("?")]
gwdata_args = url[(url.find("?") + 1):].split("&")
# First retrieve a list of urls to download from the gwdatafind server
gwdata_urls = self.get_urls(gwdatafind_server, gwdata_args)
# Now transfer each url
curl = pycurl.Curl()
for gwdata_url in gwdata_urls:
start_time = time.time()
# Setup the transfer
transfer_success = True
transfer_error = ""
file_name = gwdata_url.split("/")[-1]
file_size = 0
file = open(file_name, "wb")
# Use pycurl to actually grab the file
curl.setopt(curl.URL, gwdata_url)
curl.setopt(curl.WRITEDATA, file)
try:
curl.perform()
file_size = file.tell()
file.close()
except Exception as error:
transfer_success = False
errno, errstr = error.args
transfer_error = str(errstr + " (Error " + str(errno) + ")")
end_time = time.time()
# Add statistics for this transfer
this_transfer_stats = {
'TransferSuccess': transfer_success,
'TransferProtocol': 'gwdata',
'TransferType': 'download',
'TransferFileName': file_name,
'TransferFileBytes': file_size,
'TransferTotalBytes': file_size,
'TransferStartTime': int(start_time),
'TransferEndTime': int(end_time),
'ConnectionTimeSeconds': end_time - start_time,
'TransferUrl': gwdata_url,
}
if transfer_error:
this_transfer_stats['TransferError'] = transfer_error
transfer_stats.append(this_transfer_stats)
# If this transfer failed, exit immediately
if not transfer_success:
curl.close()
return transfer_stats, False
# Check if the args list is requesting a cache file; if so, create it
if "cache" in url:
self.create_cache(gwdata_args, gwdata_urls)
# Looks like everything downloaded correctly. Exit success.
curl.close()
return transfer_stats, True
#! /usr/bin/env python # -*- coding: utf-8 -*- # vi:ts=4:et import pycurl c = pycurl.Curl() # Redirects to https://www.python.org/. c.setopt(c.URL, 'http://www.python.org/') # Follow redirect. c.setopt(c.FOLLOWLOCATION, True) c.perform() c.close()
def _dispatch_via_pycurl(**kwds):
# import locally to allow override
from . import (
SNAPSEARCH_API_ACCEPT_ENCODING,
SNAPSEARCH_API_FOLLOW_REDIRECT,
SNAPSEARCH_API_TIMEOUT,
)
# HTTPS connection
import pycurl
c = pycurl.Curl()
c.setopt(pycurl.URL, kwds['url'])
# HTTPS POST request
headers_dict, payload = _build_message(kwds['payload'])
headers = ["%s: %s" % (key, val) for key, val in headers_dict.items()]
c.setopt(pycurl.POST, True)
c.setopt(pycurl.HTTPHEADER, headers)
c.setopt(pycurl.POSTFIELDS, payload)
# authentication
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
c.setopt(pycurl.USERPWD, "%s:%s" % (kwds['email'], kwds['key']))
# SSL verfification
c.setopt(pycurl.CAINFO, kwds['ca_path'])
c.setopt(pycurl.SSL_VERIFYPEER, True)
c.setopt(pycurl.SSL_VERIFYHOST, 2)
# transfer parameters
CURLOPT_ENCODING = getattr(pycurl, 'ACCEPT_ENCODING', pycurl.ENCODING)
c.setopt(CURLOPT_ENCODING, SNAPSEARCH_API_ACCEPT_ENCODING)
c.setopt(pycurl.FOLLOWLOCATION, SNAPSEARCH_API_FOLLOW_REDIRECT)
c.setopt(pycurl.TIMEOUT, SNAPSEARCH_API_TIMEOUT)
# buffer for response
buffer_ = bytearray()
c.setopt(pycurl.HEADER, True)
c.setopt(pycurl.WRITEFUNCTION, buffer_.extend)
try:
c.perform()
# markup buffer sections
CRLF = b"\r\n"
eos = buffer_.find(CRLF) # end of status line
eoh = buffer_.find(CRLF + CRLF) # end of header lines
# response status
normalize_status = \
lambda tup: tup[2].partition(b" ")[::2]
status_tuple = tuple(
map(lambda b: bytes(b.strip()),
normalize_status(buffer_[:eos].partition(b" "))))
status_code = int(status_tuple[0] or "0")
# response headers
normalize_header = \
lambda hdr: (bytes(hdr[0].strip().lower()),
bytes(hdr[2].strip()))
headers = dict(
map(lambda b: normalize_header(b.partition(b":")),
buffer_[eos + len(CRLF):eoh].splitlines()))
# response content
text = bytes(buffer_[eoh + 2 * len(CRLF):].strip()).decode()
except pycurl.error as e:
raise error.SnapSearchConnectionError(e)
except Exception as e:
raise error.SnapSearchError(
"malformed response from SnapSearch backend")
else:
return Response(status=status_code,
headers=headers,
body=json.loads(text))
finally:
c.close()
pass # void return
