def request(self, method, resource='', data=None, headers=None):
if self.api_url is None or self.auth_token is None or self.node_id is None:
raise SciroccoInitParamsError
method = method.upper()
url = self.get_uri(resource)
if isinstance(headers, dict):
headers.update(self.get_fixed_headers())
else:
headers = self.get_fixed_headers()
headers.update({"Content-Type": self.content_type_detector.detect_from_body(data)})
if isinstance(data, dict):
if method in ['GET', 'DELETE']:
url = ''.join([url, '?', urlencode(data)])
data = None
else:
data = json.dumps(data)
try:
request_manager_result = self.request_manager.urlopen(method, url, headers=headers, body=data)
return self.request_manager_response_handler.handle(request_manager_result)
except HTTPError as e:
SciroccoRequestAdapterError(e)
def set_query_parameters(self, parameters):
"""Set the query parameters.
:param parameters: A `dict` representing the query parameters to be used for the request.
:type parameters: dict
"""
self.query_parameters = urllib.urlencode(OrderedDict(sorted(parameters.items())))
def send_sms(text="Empty!", secured=True):
""" Sens a free SMS to the user identified by [user], with [password].
:user: Free Mobile id (of the form [0-9]{8}),
:password: Service password (of the form [a-zA-Z0-9]{14}),
:text: The content of the message (a warning is displayed if the message is bigger than 480 caracters)
:secured: True to use HTTPS, False to use HTTP.
Returns a boolean and a status string.
"""
#: Identification Number free mobile
user = '******'
#: Password
password = '******'
print("Your message is: ", text)
dictQuery = {"user": user, "pass": password, "msg": text}
url = "https" if secured else "http"
string_query = dumps(dictQuery, sort_keys=True, indent=4)
string_query = string_query.replace(password, '*' * len(password))
print("\nThe web-based query to the Free Mobile API (<u>{}://smsapi.free-mobile.fr/sendmsg?query<U>) will be based on:\n{}.".format(url, string_query))
query = urlencode(dictQuery)
url += "://smsapi.free-mobile.fr/sendmsg?{}".format(query)
try:
urlopen(url)
return 0
except HTTPError as e:
if hasattr(e, "code"):
return e.code
else:
print("Unknown error...")
return 2, "Unknown error..."
def set_query_parameters(self, parameters):
"""Set the query parameters.
:param parameters: A `dict` representing the query parameters to be used for the request.
:type parameters: dict
"""
self.query_parameters = urllib.urlencode(
OrderedDict(sorted(parameters.items())))
def build_url(self, action: str, params: Optional[Dict[str, str]] = None) -> str:
url = self.api_url + '/' + action + '?'
params = params or {}
params = {key: val for (key, val) in params.items() if val}
query = urlencode(params)
url += query + '&checksum=' + self._checksum(action, query)
return url
def _build_url(self, base_url, query_args):
"""
Builds a URL string with given query parameters
:param base_url: URL base
:type base_url: String
:param query_args: URL Query parameters
:type query_args: Dict
:returns: URL with query parameters
:rtype: String
"""
return '{}?{}'.format(base_url, urlencode(query_args))
def start_requests(self):
base_url = 'http://image.so.com/zj?'
data = {'listtype': 'new', 'temp': '1'}
ch_list = ['video', 'beauty', 'funny', 'go'] # 这个导航内容太多了少搞点
for ch in ch_list:
# print(ch)
data['ch'] = ch
# print(data)
for page in range(0, 50):
data['sn'] = page * 30
# print(data)
full_url = base_url + urlencode(data)
# print(full_url)
yield scrapy.Request(url=full_url, callback=self.parse)
def gen_sign_str(self):
if self.method == "POST":
self.algorithm = "TC3-HMAC-SHA256"
self.date = datetime.utcfromtimestamp(self.timestamp).strftime("%Y-%m-%d")
self.credential_scope = self.date + "/" + self.service + "/" + "tc3_request"
hashed_canonical_request = hashlib.sha256(self.canonical_request.encode("utf-8")).hexdigest()
string_to_sign = (self.algorithm + "\n" +
str(self.timestamp) + "\n" +
self.credential_scope + "\n" +
hashed_canonical_request)
self.string_to_sign = string_to_sign
elif self.method == "GET":
# query_string = requests.utils.urlparse(requests.get(self.url, params=self.signDictData).url).query
query_string = urlencode(self.signDictData)
self.string_to_sign = "%s%s%s%s"%(self.method, self.url.split('//')[1], "?",
query_string)
def run(self):
BASE_URL = 'http://cchart.xyz/ytcollect/YoutubeChannelTrackingService/more/?format=json¤tViewType=channelSubscribers'
response = requests.get('http://cchart.xyz/')
soup = BeautifulSoup(response.text, 'html.parser')
category_tags = soup.select('a.btn.btn-block')
cat_eng = [category.get('href').split('/')[-2] for category in category_tags]
cat_kor = [category.text.strip() for category in category_tags]
cat_dic = {eng:kor for (eng, kor) in zip(cat_eng, cat_kor)}
cat_eng_use = cat_eng[:-3]
params_list = [{'pk' :cat, 'itemTotal': number } for cat in cat_eng_use for number in page_list]
for param in params_list:
response = requests.get(BASE_URL, param)
if response.status_code == 200:
parsed_json_str = json.loads(response.text)
parsed_json = json.loads(parsed_json_str)
date_key = list(parsed_json.keys())[0]
parsed_json_01 = parsed_json[date_key]
cate_key = list(parsed_json_01.keys())[0]
parsed_json_02 = parsed_json_01[cate_key]
sub_key = list(parsed_json_02.keys())[0]
parsed_json_03 = parsed_json_02[sub_key]
rank_keys = list(parsed_json_03.keys())
for key in rank_keys:
detail = parsed_json_03[key]
youtube_id = detail['ci']
c_name = detail['ct']
category = cat_dic[param['pk']]
source_url = BASE_URL + '&' + request.urlencode(param)
json_data = {'name':c_name, 'id':youtube_id, 'category':category, 'source_url':source_url}
self.curs.execute(self.query, args=json_data)
self.conn.commit()
time.sleep(random.random() + 1)
else:
source_url = BASE_URL + '&' + urllib3.request.urlencode(param)
logging.error('%s %s' % (str(response.status_code), source_url))
self.conn.close()
def place_order(self, pair_id, amount, type, price):
nonce = str(int(time.time()) * 1e6)
uri = 'order/new'
post_data = {
'nonce': nonce,
'pair_id': pair_id,
'amount': amount,
'type': type,
'price': price
}
str_to_sign = str(urlencode(post_data))
signature = hmac.new(self.secret.encode('utf-8'),
msg=str_to_sign.encode('utf-8'),
digestmod=hashlib.sha512).hexdigest()
headers = {'Key': self.key, 'Sign': signature}
r = requests.post(self.BASE_URL + uri, data=post_data, headers=headers)
response = json.loads(r.text)
self.logger.info(self._format_log(response, 'INFO'))
return response
def place_order(self, pair_id, amount, type, price): # place a order
if self.key and self.secret:
try:
success = False
count = 0
response = ''
while not success and count < 5:
count += 1
nonce = str(int(time.time()) * 1e6)
uri = 'order/new' # Api_method/api_action
post_data = {'nonce': nonce,
'pair_id': pair_id, # Currency pair id
'amount': amount, # The volume of transactions (amount)
'type': type, # Transaction type (type) – buy / sell
'price': price # The price of the buy / sell
}
str_to_sign = str(urlencode(post_data)) # encoding post data for signature
signature = hmac.new(self.secret.encode('utf-8'), msg=str_to_sign.encode('utf-8'),
digestmod=hashlib.sha512).hexdigest()
headers = {'Content-Type': 'application/x-www-form-urlencoded',
'Key': self.key,
'Sign': signature}
log_msg = 'Trying ' + str(count) + ' time'
self.logger.info(self._format_log(log_msg, 'INFO'))
r = requests.post(self.BASE_URL + uri, data=post_data, headers=headers) # placing order on ice3x exchange
response = json.loads(r.text)
if response.get('errors') == 'false' or response.get('errors') == False:
success = True
self.logger.info(self._format_log(response, 'INFO'))
else:
self.logger.info(self._format_log(response, 'ERROR'))
time.sleep(5)
return response
except Exception as e:
self.logger.info(self._format_log(e, 'ERROR'))
return {}
else:
return "KEY AND SECRET NEEDED FOR BETTING"
def __init__(
self,
base_url='https://music.naver.com/listen/top100.nhn?domain=TOTAL&duration=1h'
):
self.base_url = base_url
self.params = [request.urlencode({'page': i}) for i in range(1, 3)]
def make_query_params(self, data):
query_params = urlencode(data)
return query_params
def test_request_get_method_data_is_same_as_url_params(self):
data_fixture = {"queryparam1": 23, "queryparam2": 34}
res = self.request_adapter.request('GET', '/resource', data_fixture)
self.assertEqual(res.http_headers['url'],
''.join([self.request_adapter.api_url, '/resource', '?', urlencode(data_fixture)]))
def parse_input(self, values):
data = req.urlencode(values)
data = urllib.parse.urlencode(values)
data = data.encode('ascii')
return (data)
def __init__(self, base_url='https://www.genie.co.kr/chart/top200?'):
self.base_url = base_url
self.params = [request.urlencode({'pg': i}) for i in range(1, 5)]
def __init__(self, base_url = 'https://www.melon.com/chart/index.htm#params%5Bidx%'):
self.base_url = base_url
self.params = request.urlencode({'5D':1})
# -*- coding:UTF-8 -*-
'''
Created on 2017年11月23日
@author: why
'''
from urllib3 import request
data = {
'bean': 'kpiInfoService',
'method': 'loadPortFlowInfo',
'p0':
'{"beginTime":"2019-01-24 10:00:00","endTime":"2019-01-24 14:10:00","kpiIds":[320,321]}',
'p1': '7',
'p2': 'NE=180002042@15',
'p3': '30000',
'p4': 'opentsdb'
}
test_data_urlencode = request.urlencode(data)
request.RequestMethods.urlopen
requrl = "http://192.168.117.161:9080/Cloud-web/callRemoteFunction/exec/kpiInfoService/loadPortFlowInfo"
req = urllib3.request(url=requrl, data=test_data_urlencode)
print(req)