def authenticate():
method = plex_settings["authentication_method"].lower()
try:
home_user_sync = plex_settings["home_user_sync"].lower()
home_username = plex_settings["home_username"]
home_server_base_url = plex_settings["home_server_base_url"]
except Exception:
home_user_sync = "false"
home_username = ""
home_server_base_url = ""
try:
session = Session()
session.mount("https://", HostNameIgnoringAdapter())
# Direct connection
if method == "direct":
base_url = plex_settings["base_url"]
token = plex_settings["token"]
plex = PlexServer(base_url, token, session)
# Myplex connection
elif method == "myplex":
plex_server = plex_settings["server"]
plex_user = plex_settings["myplex_user"]
plex_password = plex_settings["myplex_password"]
if home_user_sync == "true":
if home_username == "":
logger.error(
"Home authentication cancelled as certain home_user settings are invalid"
)
return None
logger.warning(
f"Authenticating as admin for MyPlex home user: {home_username}"
)
plex_account = MyPlexAccount(plex_user, plex_password)
plex_server_home = PlexServer(home_server_base_url,
plex_account.authenticationToken,
session)
logger.warning("Retrieving home user information")
plex_user_account = plex_account.user(home_username)
logger.warning("Retrieving user token for MyPlex home user")
plex_user_token = plex_user_account.get_token(
plex_server_home.machineIdentifier)
logger.warning("Retrieved user token for MyPlex home user")
plex = PlexServer(home_server_base_url, plex_user_token,
session)
logger.warning(
"Successfully authenticated for MyPlex home user")
else:
account = MyPlexAccount(plex_user,
plex_password,
session=session)
plex = account.resource(plex_server).connect()
else:
logger.critical(
"[PLEX] Failed to authenticate due to invalid settings or authentication info, exiting..."
)
sys.exit(1)
return plex
except Exception:
logger.exception("Unable to authenticate to Plex Media Server")
sys.exit(1)
def test_merge_headers_with_session_headers_only():
session = Session()
merged = merge_headers(session, None)
assert merged == session.headers
def test_merge_headers_with_all():
session = Session()
headers = {'Content-Type': 'test'}
merged = merge_headers(session, headers)
session.headers.update(headers)
assert merged == session.headers
def __init__(self):
self.s = Session()
self.corpid = 'wwe653983e4c732493'
self.corpsecret = 'T72_Vgw9TaNS-FLDU2gJlw6AteerMXsuMval9kGNZbc'
self.s.params["access_token"] = self.get_token().get("access_token")
def __init__(self):
self._session = Session()
def __init__(self):
self.session = Session()
self.base_path = config.BASE_DOWNLOAD_PATH if config.BASE_DOWNLOAD_PATH else 'out'
def send_raw_request(cli_ctx, method, url, headers=None, uri_parameters=None, # pylint: disable=too-many-locals,too-many-branches,too-many-statements
body=None, skip_authorization_header=False, resource=None, output_file=None,
generated_client_request_id_name='x-ms-client-request-id'):
import uuid
from requests import Session, Request
from requests.structures import CaseInsensitiveDict
result = CaseInsensitiveDict()
for s in headers or []:
try:
temp = shell_safe_json_parse(s)
result.update(temp)
except CLIError:
key, value = s.split('=', 1)
result[key] = value
headers = result
# If Authorization header is already provided, don't bother with the token
if 'Authorization' in headers:
skip_authorization_header = True
# Handle User-Agent
agents = [get_az_rest_user_agent()]
# Borrow AZURE_HTTP_USER_AGENT from msrest
# https://github.com/Azure/msrest-for-python/blob/4cc8bc84e96036f03b34716466230fb257e27b36/msrest/pipeline/universal.py#L70
_ENV_ADDITIONAL_USER_AGENT = 'AZURE_HTTP_USER_AGENT'
if _ENV_ADDITIONAL_USER_AGENT in os.environ:
agents.append(os.environ[_ENV_ADDITIONAL_USER_AGENT])
# Custom User-Agent provided as command argument
if 'User-Agent' in headers:
agents.append(headers['User-Agent'])
headers['User-Agent'] = ' '.join(agents)
if generated_client_request_id_name:
headers[generated_client_request_id_name] = str(uuid.uuid4())
# try to figure out the correct content type
if body:
try:
_ = shell_safe_json_parse(body)
if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
except Exception: # pylint: disable=broad-except
pass
# add telemetry
headers['CommandName'] = cli_ctx.data['command']
if cli_ctx.data.get('safe_params'):
headers['ParameterSetName'] = ' '.join(cli_ctx.data['safe_params'])
result = {}
for s in uri_parameters or []:
try:
temp = shell_safe_json_parse(s)
result.update(temp)
except CLIError:
key, value = s.split('=', 1)
result[key] = value
uri_parameters = result or None
endpoints = cli_ctx.cloud.endpoints
# If url is an ARM resource ID, like /subscriptions/xxx/resourcegroups/xxx?api-version=2019-07-01,
# default to Azure Resource Manager.
# https://management.azure.com + /subscriptions/xxx/resourcegroups/xxx?api-version=2019-07-01
if '://' not in url:
url = endpoints.resource_manager.rstrip('/') + url
# Replace common tokens with real values. It is for smooth experience if users copy and paste the url from
# Azure Rest API doc
from azure.cli.core._profile import Profile
profile = Profile(cli_ctx=cli_ctx)
if '{subscriptionId}' in url:
url = url.replace('{subscriptionId}', cli_ctx.data['subscription_id'] or profile.get_subscription_id())
# Prepare the Bearer token for `Authorization` header
if not skip_authorization_header and url.lower().startswith('https://'):
# Prepare `resource` for `get_raw_token`
if not resource:
# If url starts with ARM endpoint, like `https://management.azure.com/`,
# use `active_directory_resource_id` for resource, like `https://management.core.windows.net/`.
# This follows the same behavior as `azure.cli.core.commands.client_factory._get_mgmt_service_client`
if url.lower().startswith(endpoints.resource_manager.rstrip('/')):
resource = endpoints.active_directory_resource_id
else:
from azure.cli.core.cloud import CloudEndpointNotSetException
for p in [x for x in dir(endpoints) if not x.startswith('_')]:
try:
value = getattr(endpoints, p)
except CloudEndpointNotSetException:
continue
if isinstance(value, str) and url.lower().startswith(value.lower()):
resource = value
break
if resource:
# Prepare `subscription` for `get_raw_token`
# If this is an ARM request, try to extract subscription ID from the URL.
# But there are APIs which don't require subscription ID, like /subscriptions, /tenants
# TODO: In the future when multi-tenant subscription is supported, we won't be able to uniquely identify
# the token from subscription anymore.
token_subscription = None
if url.lower().startswith(endpoints.resource_manager.rstrip('/')):
token_subscription = _extract_subscription_id(url)
if token_subscription:
logger.debug('Retrieving token for resource %s, subscription %s', resource, token_subscription)
token_info, _, _ = profile.get_raw_token(resource, subscription=token_subscription)
else:
logger.debug('Retrieving token for resource %s', resource)
token_info, _, _ = profile.get_raw_token(resource)
token_type, token, _ = token_info
headers = headers or {}
headers['Authorization'] = '{} {}'.format(token_type, token)
else:
logger.warning("Can't derive appropriate Azure AD resource from --url to acquire an access token. "
"If access token is required, use --resource to specify the resource")
# https://requests.readthedocs.io/en/latest/user/advanced/#prepared-requests
s = Session()
req = Request(method=method, url=url, headers=headers, params=uri_parameters, data=body)
prepped = s.prepare_request(req)
# Merge environment settings into session
settings = s.merge_environment_settings(prepped.url, {}, None, not should_disable_connection_verify(), None)
_log_request(prepped)
r = s.send(prepped, **settings)
_log_response(r)
if not r.ok:
reason = r.reason
if r.text:
reason += '({})'.format(r.text)
raise CLIError(reason)
if output_file:
with open(output_file, 'wb') as fd:
for chunk in r.iter_content(chunk_size=128):
fd.write(chunk)
return r
def use_wrapper():
print("Using helper")
sess = CacheControl(Session())
return sess
class RequestsManager:
"""Base manager used to send requests to the internet
"""
session = Session()
request_errors = []
@classmethod
def get(cls, urls:list, **kwargs):
threads = []
responses = []
with cls.session as new_session:
def new_session_wrapper(request):
response = new_session.send(request)
if response.status_code == 200:
# TODO: Apparently the deque()
# mutates. Maybe we need to create
# and instance in the method
responses.append(response)
else:
cls.update(cls, url)
for index, url in enumerate(urls):
if isinstance(url, (tuple, list)):
url = url[0]
if not url.startswith('http'):
raise ValueError(f'Url should start with http or https. In the case where you are using a tuple, the url should be at position 0. Received: {url[0]}')
thread = threading.Thread(
target=new_session_wrapper,
args=[cls.prepare(cls, url, **kwargs)]
)
threads.append(thread)
if 'limit_to' in kwargs:
limit_to = kwargs['limit_to']
has_reached_limit = all([index == limit_to, limit_to > 0])
if has_reached_limit:
break
if threads:
for thread in threads:
thread.start()
if thread.is_alive():
print('GET HTTP/1.1', '--')
thread.join()
if len(urls) == 1:
return responses[0]
return responses
def prepare(self, url, **headers):
"""This definition prepares a request to send to the web.
This definition was structured so that the prepared request
can be modified until the last moment.
"""
base_headers = {
'User-Agent': user_agent.get_rand_agent()
}
if headers:
base_headers = {**base_headers, **headers}
request = Request(method='GET', url=url, headers=headers)
prepared_request = self.session.prepare_request(request)
return prepared_request
def update(self, url, **kwargs):
"""Updates the request error stack
"""
return self.request_errors.append((url, Error('The request with "%s" was not successful' % url)))
def beautify(self, urls:list, **kwargs):
"""Returns BeautifulSoup objects
"""
soups = []
responses = self.get(urls, **kwargs)
for response in responses:
soups.append(BeautifulSoup(response.text, 'html.parser'))
return soups
def beautify_single(self, url):
"""Returns a BeautifulSoup object
"""
response = self.get([url])
return BeautifulSoup(response.text, 'html.parser')
def do_test_hot_client(content_object_id):
"""发送前端热更新请求到管理机
走本地的celery worker
"""
try:
# cmdb上面的错误 用于cmdb出错以后发送邮件报警
CMDB_ERROR = False
content_object = ClientHotUpdate.objects.get(id=content_object_id)
hot_update_log = HotUpdateLog(content_object.uuid)
# 首先去检查lock项目和地区，如果lock失败，直接引发一个异常
"""2019.3修改，运维管理机通过工单子任务表RsyncTask来获取"""
list_ops_manager_id = [x.ops_id for x in content_object.clienthotupdatersynctask_set.all()]
"""找到其他url相同的运维管理机，事实为同一台管理机，一起上锁"""
all_list_ops_manager_id = []
for ops_id in list_ops_manager_id:
ops_obj = OpsManager.objects.get(pk=ops_id)
url = ops_obj.url
for x in OpsManager.objects.filter(url__icontains=url):
all_list_ops_manager_id.append(x.id)
list_ops_manager = OpsManager.objects.filter(id__in=all_list_ops_manager_id)
list_ops_manager_status = [x.status for x in list_ops_manager]
if not (len(list(set(list_ops_manager_status))) == 1 and '0' in list_ops_manager_status):
content_object.status = '4'
content_object.save()
raise HotUpdateBlock('%s: 项目和地区已经被锁,进入待更新状态' % content_object.title)
# hot_update_log.logger.info('%s: 项目和地区已经被锁,进入待更新状态' % (content_object.title))
else:
# lock
hot_update_log.logger.info('%s: 开始执行' % content_object.title)
update_status = {'status': '2'}
list_ops_manager.update(**update_status)
hot_update_log.logger.info('hot_client: %s-成功上锁' % content_object.title)
"""
2019.3修改
遍历热更新的子任务，依次发送对应的热更新数据到相应的运维管理机
"""
for task in content_object.clienthotupdatersynctask_set.all():
content = json.loads(task.content)
update_file_list = json.loads(task.update_file_list)
update_file_list = format_hot_update_file_list(update_file_list, 'area_dir')
hot_client_data = format_hot_client_data(
content, content_object.uuid, content_object.client_type, update_file_list,
content_object.client_version)
# 获取运维管理机
ops_manager = task.ops
# 发送前端热更新数据给运维管理机
url = ops_manager.url + CLIENT_HOT
token = ops_manager.token
authorized_token = "Token " + token
headers = {
'Accept': 'application/json',
'Authorization': authorized_token,
'Connection': 'keep-alive',
}
s = Session()
s.mount('https://', HTTPAdapter(max_retries=Retry(total=3, status_forcelist=[408])))
# r = s.post(url, headers=headers, json=hot_client_data, verify=False, timeout=10)
# result = r.json()
result = {'Accepted': True}
if result.get('Accepted', False):
hot_update_log.logger.info('hot_client: %s-发送消息到管理机%s成功' % (content_object.title, ops_manager))
else:
content_object.status = '2'
content_object.save()
hot_update_log.logger.error('hot_client: %s-发送消息到管理机%s失败' % (content_object.title, ops_manager))
raise Exception('hot_client: %s-发送消息到管理机%s失败' % (content_object.title, ops_manager))
except OpsManager.DoesNotExist:
msg = 'hot_client: ops manager not found'
hot_update_log.logger.error('%s' % msg)
content_object.status = '2'
content_object.save()
CMDB_ERROR = True
except requests.exceptions.ConnectionError:
content_object.status = '2'
content_object.save()
CMDB_ERROR = True
hot_update_log.logger.error('hot_client: %s: %s time out' % (content_object.title, url))
except GameProject.DoesNotExist:
msg = 'hot_client: game project not found'
hot_update_log.logger.error('%s' % msg)
content_object.status = '2'
content_object.save()
CMDB_ERROR = True
except Room.DoesNotExist:
msg = 'hot_client: room not found'
hot_update_log.logger.error('%s' % (msg))
content_object.status = '2'
content_object.save()
CMDB_ERROR = True
except HotUpdateBlock as e:
msg = str(e)
hot_update_log.logger.error('%s' % (msg))
CMDB_ERROR = True
except Exception as e:
msg = str(e)
hot_update_log.logger.error('%s' % (msg))
content_object.status = '2'
content_object.save()
CMDB_ERROR = True
finally:
# 通知客户端刷新状态
content_object.save()
ws_notify()
if CMDB_ERROR:
pass
for task in content_object.clienthotupdatersynctask_set.all():
client_hotupdate_callback(uuid=content_object.uuid, client_version=content_object.client_version,
client_task=task)
def use_adapter():
print("Using adapter")
sess = Session()
sess.mount("http://", CacheControlAdapter())
return sess
def do_test_hot_server(content_object_id):
"""执行热更新后端
"""
content_object = ServerHotUpdate.objects.get(id=content_object_id)
hot_update_log = HotUpdateLog(content_object.uuid)
try:
# cmdb上面的错误 用于cmdb出错以后发送邮件报警
CMDB_ERROR = False
# 首先去检查lock项目和地区，如果lock失败，直接引发一个异常
"""2019.3修改，运维管理机通过工单子任务表RsyncTask来获取"""
list_ops_manager_id = [x.ops_id for x in content_object.serverhotupdatersynctask_set.all()]
"""找到其他url相同的运维管理机，事实为同一台管理机，一起上锁"""
all_list_ops_manager_id = []
for ops_id in list_ops_manager_id:
ops_obj = OpsManager.objects.get(pk=ops_id)
url = ops_obj.url
for x in OpsManager.objects.filter(url__icontains=url):
all_list_ops_manager_id.append(x.id)
list_ops_manager = OpsManager.objects.filter(id__in=all_list_ops_manager_id)
list_ops_manager_status = [x.status for x in list_ops_manager]
if not (len(list(set(list_ops_manager_status))) == 1 and '0' in list_ops_manager_status):
content_object.status = '4'
content_object.save()
task = content_object.serverhotupdatersynctask_set.all()
task.update(**{'rsync_result': None})
raise HotUpdateBlock('%s: 项目和地区已经被锁,进入待更新状态' % (content_object.title))
# hot_update_log.logger.info('%s: 项目和地区已经被锁,进入待更新状态' % (content_object.title))
else:
# lock
hot_update_log.logger.info('%s: 开始执行' % (content_object.title))
update_status = {'status': '2'}
list_ops_manager.update(**update_status)
hot_update_log.logger.info('hot_server: %s-成功上锁' % (content_object.title))
# 进行一次区服的数据校验（增加新服）
revise_server_list(content_object)
"""区服数据校验后，重新更新子任务对应的区服数据"""
update_server_list = json.loads(content_object.update_server_list)
for update_server in update_server_list:
game_server_id = update_server['gameserverid']
game_server = GameServer.objects.get(pk=int(game_server_id))
ops = game_server.host.opsmanager
if ops:
"""判断管理机对应的子任务是否已存在，是则更新字段，否则创建子任务"""
task = ServerHotUpdateRsyncTask.objects.filter(server_hot_update=content_object, ops=ops)
if task:
task_update_server_list = json.loads(task[0].update_server_list)
if update_server not in task_update_server_list:
task_update_server_list.append(update_server)
task_update_server_list = json.dumps(task_update_server_list)
task.update(**{"update_server_list": task_update_server_list})
# 修改final_result最终结果为None
content_object.final_result = None
# 复制update_server_list到result_update_file_list
result_update_file_list = content_object.update_server_list
content_object.result_update_file_list = result_update_file_list
content_object.save()
hot_update_log.logger.info('hot_server: %s-复制数据成功' % (content_object.title))
# 加载热更新区服列表到redis中
load_to_redis(content_object)
hot_update_log.logger.info('hot_server: %s-将数据load到redis成功' % (content_object.title))
"""
2019.3修改，遍历热更新子任务，发送对应的更新区服数据到对应的运维管理机
"""
for task in content_object.serverhotupdatersynctask_set.all():
# 热更新后端数据
update_server_list = format_hot_server_data(json.loads(task.update_server_list))
# 获取运维管理机
ops_manager = task.ops
url = ops_manager.url + CLIENT_HOT
token = ops_manager.token
authorized_token = "Token " + token
headers = {
'Accept': 'application/json',
'Authorization': authorized_token,
'Connection': 'keep-alive',
}
# 后端热更新需要post的数据
hot_server_data = {}
hot_server_data['uuid'] = content_object.uuid
hot_server_data['update_type'] = 'hot_server'
hot_server_data['update_server_list'] = update_server_list
hot_server_data['version'] = content_object.server_version
hot_server_data['hot_server_type'] = content_object.hot_server_type
if content_object.erlang_cmd_list:
hot_server_data['erlang_cmd_list'] = content_object.erlang_cmd_list.split('\n')
else:
hot_server_data['erlang_cmd_list'] = []
if content_object.update_file_list:
hot_server_data['update_file_list'] = format_hot_update_file_list(json.loads(task.update_file_list),
'area_dir')
else:
hot_server_data['update_file_list'] = []
# r = requests.post(url, headers=headers, json=hot_server_data, verify=False, timeout=30)
s = Session()
s.mount('https://', HTTPAdapter(max_retries=Retry(total=3, status_forcelist=[408])))
# r = s.post(url, headers=headers, json=hot_server_data, verify=False, timeout=10)
# result = r.json()
result = {"Accepted": True}
if result.get('Accepted', False):
hot_update_log.logger.info('hot_server: %s-发送消息到管理机%s成功' % (content_object.title, ops_manager))
server_hotupdate_callback(content_object.uuid, content_object.server_version, task)
else:
hot_update_log.logger.error('hot_server: %s-发送消息到管理机%s失败' % (content_object.title, ops_manager))
raise Exception('hot_server: %s-发送消息到管理机%s失败' % (content_object.title, ops_manager))
server_hotupdate_callback_on_finish(content_object.uuid, content_object.server_version)
except OpsManager.DoesNotExist:
msg = 'hot_server: 项目%s-地区%s没有找到运维管理机' % (content_object.project.project_name, content_object.area_name)
hot_update_log.logger.error('%s' % (msg))
content_object.status = '2'
content_object.save()
task = content_object.serverhotupdatersynctask_set.all()
task.update(**{'rsync_result': None})
CMDB_ERROR = True
except requests.exceptions.ConnectionError:
content_object.status = '2'
content_object.save()
task = content_object.serverhotupdatersynctask_set.all()
task.update(**{'rsync_result': None})
CMDB_ERROR = True
hot_update_log.logger.error('hot_server: %s: %s time out' % (content_object.title, url))
except HotUpdateBlock as e:
msg = str(e)
hot_update_log.logger.error('%s' % (msg))
CMDB_ERROR = True
except Exception as e:
msg = 'hot_server: ' + str(e)
hot_update_log.logger.error('%s' % (msg))
content_object.status = '2'
content_object.save()
task = content_object.serverhotupdatersynctask_set.all()
task.update(**{'rsync_result': None})
CMDB_ERROR = True
finally:
ws_notify()
class Http():
http = Session()
path = 'top_f.txt'
family = open(path, 'r')
family_list = []
for i in family:
family_list.append(i.replace('\n', ''))
print('Surnames uploaded successfully.')
user_agent = ('Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) '
'Gecko/20100101 Firefox/50.0')
headers={
'User-Agent':user_agent,
'Referer': 'https://pra.in.ua/en/'
}
my_sesion = Session()
response = my_sesion.get(
url='https://pra.in.ua/en/login',
headers={'User-Agent':user_agent},
)
regex = r"name=\"_token\" value=\"(.+)\""
short_token = search(regex, response.text).groups()[0]
response = my_sesion.post(
url='https://pra.in.ua/en/login',
headers=headers,
data={
'email': '*****@*****.**',
def __init__(self, proxies, recheck_interval=60, max_retries=2):
self.proxies = list(proxies.values())
self.session = Session()
self.recheck_interval = recheck_interval
self.max_retries = max_retries
def initialize(self):
self.session = Session()
self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__
def session(self):
s = Session()
s.mount('http://',
HTTPAdapter(max_retries=Retry(total=100, backoff_factor=0.1)))
return s
def send_request(self, endpoint, method, jsondata=None, params=None):
"""
Common function that all individual DEP commands are passed through,
prepped (via dep_prep()), sent to the DEP API and relays responses back to
the caller.
Required parameters:
- endpoint (The DEP API endpoint for a specific query)
- method (The HTTP method to use for the query - GET/POST/PUT)
Optional parameters:
- jsondata (Any JSON data required for the API query)
- params (Any URL query parameters required for the API query)
"""
# We might need to update the auth_session_token and auth_timestamp vars at
# some point during the interaction with the DEP API so we enable them as
# globals so that all future calls have access to their updated values.
# global auth_session_token
# global auth_timestamp
s = Session()
# Check whether we received JSON data as part of the query, in which case
# we include it in the Session.request() call using the 'json' var.
if jsondata:
print("Sending JSON data to %s via %s" % (endpoint, method))
prepped = self.dep_prep(endpoint,
method,
token=self.auth_session_token)
response = s.request(method=prepped.method,
url=prepped.url,
headers=prepped.headers,
json=jsondata)
# Check whether we received HTTP query data, in which case we need to
# include it in the Session.request() call using the 'params' var.
elif params:
print("Sending query string %s to %s via %s" %
(params, endpoint, method))
prepped = self.dep_prep(endpoint,
method,
token=self.auth_session_token,
params=True)
response = s.request(method=prepped.method,
url=prepped.url,
headers=prepped.headers,
params=params)
# If we got neither JSON or query parameters we can use the less involved
# Session.send() method instead.
else:
print("Sending %s call via %s" % (endpoint, method))
prepped = self.dep_prep(endpoint,
method,
token=self.auth_session_token)
response = s.send(prepped)
# Check the status code returned in the response, if it's 401 or 403 our
# auth session expired or wasn't accepted so we request a new one and retry
# the same HTTP request again. Otherwise we move on.
if response.status_code == 401 or response.status_code == 403:
print("Token expired, updating")
self.auth_session_token, self.auth_timestamp = self.get_auth_token(
)
return self.send_request(endpoint,
method,
jsondata=jsondata,
params=params)
else:
# Try to parse the response as JSON since the majority of responses are
# formatted as JSON. If that fails we return the raw content.
# The caller will have to implement doing something useful with the
# returned content in either case.
try:
return response.json()
except ValueError:
return response.content
def initialize(self):
self.session = Session()
self.session.headers['User-Agent'] = (
'SubDB/1.0 (subliminal/%s; https://github.com/Diaoul/subliminal)' %
__short_version__)
"""
一个疑似河南郑州大学的网站接口，使用方法可参照该网站
http://122.114.198.116/
"""
from requests import get, post, Session
from urllib.parse import urlencode
import base64
session = Session()
captchaurl = 'https://kyfw.12306.cn/passport/captcha/captcha-image?login_site=E&module=login&rand=sjrand'
captchaheaders = {
'Accept':
'image / webp, image / apng, image / *, * / *;q = 0.8',
'Accept-Encoding':
'gzip, deflate,br',
'Accept-Language':
'zh-CN,zh;q=0.9',
'Host':
'kyfw.12306.cn',
'Referer':
'https://kyfw.12306.cn/otn/leftTicket/init',
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36",
}
checkcodecontent = session.get(captchaurl, headers=captchaheaders)
with open('captcha-image.jpg', 'wb') as f:
f.write(checkcodecontent.content)
session2 = Session()
url = 'http://122.114.198.116/12306/getResult'
urlheaders = {
from pyramid.exceptions import URLDecodeError
from pyramid.compat import decode_path_info
from binascii import hexlify, unhexlify
from Crypto.Cipher import AES
from re import compile
from requests import Session
PKG = get_distribution(__package__)
LOGGER = getLogger(PKG.project_name)
VERSION = '{}.{}'.format(int(PKG.parsed_version[0]), int(PKG.parsed_version[1]) if PKG.parsed_version[1].isdigit() else 0)
ROUTE_PREFIX = '/api/{}'.format(VERSION)
DOCUMENT_BLACKLISTED_FIELDS = ('title', 'format', 'url', 'dateModified', 'hash')
DOCUMENT_WHITELISTED_FIELDS = ('id', 'datePublished', 'author', '__parent__')
ACCELERATOR_RE = compile(r'.accelerator=(?P<accelerator>\d+)')
SESSION = Session()
json_view = partial(view, renderer='json')
def route_prefix(settings={}):
return '/api/{}'.format(settings.get('api_version', VERSION))
def generate_id():
return uuid4().hex
def generate_tender_id(ctime, db, server_id=''):
key = ctime.date().isoformat()
tenderIDdoc = 'tenderID_' + server_id if server_id else 'tenderID'
while True:
def detail(self, response):
item = response.meta['item']
product_url = item['product_url']
# text=product_text
'''检测到网页长度不足证明网页内容不全，重新获取'''
if len(response.text) < 65000:
yield scrapy.Request(url=product_url,
callback=self.detail,
meta=response.meta,
dont_filter=True)
return None
# 商品ID
try:
ProductID = Selector(response).re('prdId:"(.*?)"')[0]
except:
try:
ProductID = re.findall('prdId:"(.*?)"', response.text)[0]
except:
try:
ProductID = re.findall('prdId:"(.*?)"', response.text)[0]
except:
ProductID = response.url.split('/')[-1].split('-')[0]
# 好评，差评等信息采集
comment_url = 'http://ss.gome.com.cn/item/v1/prdevajsonp/appraiseNew/%s/1/all/0/10/flag/appraise' % ProductID
mark_url = 'http://ss.gome.com.cn/item/v1/prdevajsonp/productEvaComm/%s/flag/appraise/totleMarks?callback=totleMarks' % ProductID
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
}
request_retry = requests.adapters.HTTPAdapter(max_retries=3)
with Session() as gome:
#建立会话对象
gome.mount('http://', request_retry)
gome.mount('https://', request_retry)
gome.headers = headers
comment_text = gome.get(url=comment_url).text
time.sleep(0.3)
mark_text = gome.get(url=mark_url).text
#店铺名称
try:
shop_name = response.xpath(
".//div[@class='zy-stores shops-name']/a[@class='name']/text()"
).extract()[0]
except:
try:
shop_name = response.xpath(
".//h2[@id='store_live800_wrap']/a[@class='name']/text()"
).extract()[0]
except:
try:
shop_name = response.xpath(
".//div[@class='zy-stores shops-name']/span[@class='identify']/text()"
).extract()[0]
except:
shop_name = None
#价格类代码重写
try:
Price = re.findall('price:"(.*?)"', response.text)
gomeprice = re.findall('gomePrice:"(.*?)"', response.text)
groupprice = re.findall('groupPrice:"(.*?)"', response.text)
oldprice = re.findall('<span id="listPrice">(.*?)</span>',
response.text)
if Price:
if Price[0] == '0':
price = gomeprice[0]
PreferentialPrice = gomeprice[0]
else:
if float(Price[0]) < float(gomeprice[0]):
price = gomeprice[0]
PreferentialPrice = Price[0]
else:
price = Price[0]
PreferentialPrice = gomeprice[0]
else:
if float(oldprice[0]) < float(groupprice[0]):
price = groupprice[0]
PreferentialPrice = oldprice[0]
else:
price = oldprice[0]
PreferentialPrice = groupprice[0]
if float(price) < float(PreferentialPrice):
print('错误！！！')
except:
price = None
PreferentialPrice = None
#品牌
try:
brand = Selector(response).re('品牌：(.*?)</div>')[0]
except:
try:
brand = re.findall('品牌：(.*?)</div>', response.text)[0]
except:
brand = None
if brand:
if re.findall(r'（.*?）', brand):
re_com = re.compile('（.*?）')
brand = brand[:0] + re.sub(re_com, '', brand)
if brand:
if re.findall(r'(.*?)', brand):
re_cn = re.compile('\(.*?\)')
brand = brand[:0] + re.sub(re_cn, '', brand)
#品牌型号
try:
X_name = Selector(response).re('型号：(.*?)</div>')[0]
except:
try:
X_name = re.findall('型号：(.*?)</div>', response.text)[0]
except:
try:
X_name = Selector(response).re(
'型号</span><span>(.*?)</span>')[0]
except:
try:
X_name = re.findall('型号</span><span>(.*?)</span>',
response.text)[0]
except:
X_name = None
if X_name:
if brand:
if brand in X_name:
pass
else:
X_name = brand + X_name[:]
# 颜色
try:
color = Selector(response).re('颜色</span><span>(.*?)</span>')[0]
except:
try:
color = re.findall('颜色</span><span>(.*?)</span>',
response.text)[0]
except:
color = None
#功能，加热方式，控制方式，预约方式
try:
jiare = Selector(response).re('加热方式：(.*?)</div>')[0]
except:
try:
jiare = Selector(response).re(
'加热方式</span><span>(.*?)</span>')[0]
except:
try:
jiare = re.findall('加热方式：(.*?)</div>', response.text)[0]
except:
try:
jiare = re.findall('加热方式</span><span>(.*?)</span>',
response.text)[0]
except:
jiare = None
try:
kongzhi = Selector(response).re('控制方式：(.*?)</div>')[0]
except:
try:
kongzhi = Selector(response).re(
'控制方式</span><span>(.*?)</span>')[0]
except:
try:
kongzhi = re.findall('控制方式：(.*?)</div>', response.text)[0]
except:
try:
kongzhi = re.findall('控制方式</span><span>(.*?)</span>',
response.text)[0]
except:
kongzhi = None
try:
yuyue = Selector(response).re('预约方式：(.*?)</div>')[0]
except:
try:
yuyue = Selector(response).re(
'预约方式</span><span>(.*?)</span>')[0]
except:
try:
yuyue = re.findall('预约方式：(.*?)</div>', response.text)[0]
except:
try:
yuyue = re.findall('预约方式</span><span>(.*?)</span>',
response.text)[0]
except:
yuyue = None
if jiare or kongzhi or yuyue:
X_type = ''
if jiare:
X_type = X_type[:] + jiare + ' '
if kongzhi:
X_type = X_type[:] + kongzhi + ' '
if yuyue:
X_type = X_type[:] + yuyue
if len(X_type) < 1:
X_type = None
else:
X_type = None
#容量
try:
capacity = Selector(response).re('容量：(.*?)</div>')[0]
except:
try:
capacity = Selector(response).re(
'容量</span><span>(.*?)</span>')[0]
except:
try:
capacity = re.findall('容量：(.*?)</div>', response.text)[0]
except:
try:
capacity = re.findall('容量</span><span>(.*?)</span>',
response.text)[0]
except:
capacity = None
#####核心参数，特别处理部分
soup = BeautifulSoup(response.text, 'lxml')
try:
parameter = []
div_item = soup.find_all('div', class_='param-item')
# print(div_item)
for each in div_item:
li_list = each.find_all('li')
for each in li_list:
li_text = re.sub(r'\n', '', each.text)
parameter.append(li_text)
if len(parameter) < 1:
print(1 / 0)
except:
try:
parameter = []
div_item = soup.find('div', class_='guigecanshu_wrap')
div_canshu = div_item.find_all('div', class_='guigecanshu')
for each in div_canshu:
parameter.append(each.text)
if len(parameter) < 1:
print(1 / 0)
except: # 针对真划算页面的type参数分析
try:
parameter = []
table = soup.find('table', attrs={'class': 'grd-specbox'})
tr_list = table.find_all('tr')
for each in tr_list:
if each.find('td'):
td = each.find_all('td')
if td:
td1 = re.sub(r'\n', '', td[0].text)
td2 = re.sub(r'\n', '', td[1].text)
parameter.append(td1 + ':' + td2)
# print(td1 + ':' + td2)
print(parameter)
if len(parameter) < 1:
print(1 / 0)
except:
parameter = None
# 将核心参数转化为字符串形式
try:
if parameter == None:
type = None
else:
type = '"'
for i in range(len(parameter)):
type = type[:] + parameter[i]
if i < len(parameter) - 1:
type = type[:] + ' '
if i == len(parameter) - 1:
type = type[:] + '"'
except:
type = None
if type:
if brand == None:
try:
brand = re.findall('品牌:(.*?) ', type)[0]
except:
brand = None
if brand:
if re.findall(r'（.*?）', brand):
re_com = re.compile('（.*?）')
brand = brand[:0] + re.sub(re_com, '', brand)
if brand:
if re.findall(r'(.*?)', brand):
re_cn = re.compile('\(.*?\)')
brand = brand[:0] + re.sub(re_cn, '', brand)
if X_name == None:
try:
X_name = re.findall('型号:(.*?) ', type)[0]
except:
X_name = None
if X_name:
if brand:
if brand in X_name:
pass
else:
X_name = brand + X_name[:]
if color == None:
try:
color = re.findall('颜色:(.*?) ', type)[0]
except:
color = None
'''
以下为功能部分获取
'''
if X_type == None:
try:
jiare = re.findall('加热方式:(.*?) ', type)[0]
except:
jiare = None
try:
kongzhi = re.findall('控制方式:(.*?) ', type)[0]
except:
kongzhi = None
try:
yuyue = re.findall('预约方式:(.*?) ', type)[0]
except:
yuyue = None
if jiare or kongzhi or yuyue:
X_type = ''
if jiare:
X_type = X_type[:] + jiare + ' '
if kongzhi:
X_type = X_type[:] + kongzhi + ' '
if yuyue:
X_type = X_type[:] + yuyue
if len(X_type) < 1:
X_type = None
else:
X_type = None
'''
容量
'''
if capacity == None:
try:
capacity = re.findall('容量:(.*?) ', type)[0]
except:
capacity = None
# 访问comment_url
try:
# 好评
GoodCount = re.findall('"good":(.*?),', comment_text)[0]
except:
GoodCount = None
# 中评
try:
GeneralCount = re.findall('"mid":(.*?),', comment_text)[0]
except:
GeneralCount = None
# 差评
try:
PoorCount = re.findall('"bad":(.*?),', comment_text)[0]
except:
PoorCount = None
# 总评
try:
CommentCount = re.findall('"totalCount":(.*?),', comment_text)[0]
except:
CommentCount = None
# 访问评论关键字
# 好评度
try:
GoodRateShow = re.findall(r'"goodCommentPercent":(\d+)',
mark_text)[0]
except:
try:
GoodRateShow = re.findall(r'"good":(\d+),', mark_text)[0]
except:
GoodRateShow = None
try:
keyword = '"'
word_list = re.findall('"recocontent":"(.*?)"', mark_text)
for each in word_list:
if '?' in each:
word_list.remove(each)
if word_list:
for every in word_list:
keyword = keyword[:] + every
if every != word_list[-1]:
keyword = keyword[:] + ' '
if every == word_list[-1]:
keyword = keyword[:] + '"'
if len(keyword) <= 1:
print(1 / 0)
except:
keyword = None
source = '国美'
if type == None and brand == None and X_name == None:
print('一条数据被过滤！')
yield None
else:
item['ProductID'] = ProductID
item['X_name'] = X_name
item['type'] = type
item['X_type'] = X_type
item['price'] = price
item['PreferentialPrice'] = PreferentialPrice
item['brand'] = brand
item['keyword'] = keyword
item['PoorCount'] = PoorCount
item['CommentCount'] = CommentCount
item['GoodCount'] = GoodCount
item['GeneralCount'] = GeneralCount
item['GoodRateShow'] = GoodRateShow
item['shop_name'] = shop_name
item['capacity'] = capacity
item['color'] = color
item['source'] = source
yield item
from requests import Session
from zeep import Client
from zeep.transports import Transport
session = Session()
session.verify = False
client = Client('https://10.16.118.52:9443/services/AuthenticationAdmin?wsdl',
transport=Transport(session=session))
# client = Client('../wsdl/AuthenticationAdmin.xml',
# transport=Transport(session=session))
service = client.create_service(
'{http://authentication.services.core.carbon.wso2.org}AuthenticationAdminSoap11Binding',
'https://10.16.118.52:9443/services/AuthenticationAdmin')
print service.login('admin', 'admin', 'localhost')
# print client.service.login('admin', 'admin', 'localhost')
# print client.service.login('zhanlif', '123456', 'localhost')
def main(args):
autofetch = True
first = 0
second = 0
if len(args) < 1:
print "Usage monitor.py \"path to log public key\" [sth treesize 1] [sth treesize 2]"
sys.exit(1)
elif len(args) > 1:
autofetch = False
first = args[1]
second = args[2]
logkeypath = args[0]
if autofetch == True:
operation = 'ct/v1/get-sth'
url = 'http://ct.googleapis.com/aviator/{}'.format(operation)
s = Session()
r = Request('GET', url)
prepped = r.prepare()
proxies = {
#"http": "http://127.0.0.1:8080",
}
r = s.send(prepped, proxies=proxies)
numcerts = 0
if r.status_code == 200:
sth = r.json()
numcerts = sth['tree_size']
logkey = open(logkeypath, 'r').read()
if verify_sth(sth, logkey) == False:
print 'Invalid log; signed tree head failed validation'
return 1
else:
sth1 = sth
# Get another STH, keep asking until we get another one different
cacheval = re.search('\d+', r.headers['cache-control']).group(0)
fetchct = 1
delayval = 1
while True:
time.sleep(int(cacheval))
print "STH fetch # {0}".format(fetchct)
fetchct += 1
r = s.send(prepped, proxies=proxies)
if r.status_code == 200:
sth = r.json()
if verify_sth(sth, logkey) == False:
print 'Invalid log; signed tree head failed validation'
return 1
else:
sth2 = sth
delayval = int(math.log(fetchct))
if sth1['timestamp'] != sth2['timestamp']:
break
first = sth1['treesize']
second = sth1['treesize']
# Verify STHs by fetching consistency proof
operation = 'ct/v1/get-sth-consistency'
url = 'http://ct.googleapis.com/aviator/{}'.format(operation)
s = Session()
params = urllib.urlencode({'first': first, 'second': second})
r = Request('GET', '{}?{}'.format(url, params))
prepped = r.prepare()
r = s.send(prepped)
if r.status_code == 200:
print r.text
userAgent = None
userAgents = {
'chrome':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
'chrome39':
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
}
body = None
if body:
body = body.encode()
last_http_response = None
last_json_response = None
last_response_headers = None
userAgent = default_user_agent()
timeout = 10000 # milliseconds = seconds * 1000
session = session if session else Session()
response = session.request('GET',
api,
data=body,
headers=None,
timeout=int(timeout / 1000))
last_http_response = response.text
result = handle_rest_response(last_http_response)
db = pymysql.connect("localhost", "root", "KeYpZrZx", "btc")
cursor = db.cursor()
cursor.execute("truncate table coinmarket_last")
db.commit()
for data in result:
if data['price_usd'] and data['24h_volume_usd'] and data['last_updated']:
sql = """INSERT INTO coinmarket(coinid,symbol,price_usd,volume,last_updated,update_time)
values ('%s','%s',%s,%s,%s,UNIX_TIMESTAMP() )""" % (
def __init__(self, max_timeout=None, proxy=None):
self.session = Session()
self.max_timeout = max_timeout or 10000
self.proxy = proxy
self.debug = None
def test_merge_headers_with_all_none():
session = Session()
session.headers = None
merged = merge_headers(session, None)
assert merged == {}
import unittest
import vcr
from canonicalwebteam.blog import wordpress_api as api
from canonicalwebteam.blog.common_view_logic import (
get_complete_article,
get_index_context,
get_group_page_context,
get_topic_page_context,
get_article_context,
BlogViews,
)
from requests import Session
api.api_session = Session()
class TestCommonViewLogic(unittest.TestCase):
@vcr.use_cassette("fixtures/vcr_cassettes/get_complete_article.yaml")
def test_get_complete_article(self):
article = api.get_article("canonical-announces-support-for"
"-ubuntu-on-windows-subsystem-for-linux-2")
complete_article = get_complete_article(article)
self.assertEqual(complete_article["author"]["name"], "Canonical")
self.assertEqual(complete_article["image"]["id"],
article["featured_media"])
self.assertEqual(complete_article["date"], "6 May 2019")
# The maximum allowed character count for the excerpt is 339
# characters. If this limit is reached [...] is added
#BODOAMAT
#SkyFish
import time, re, sys
from requests import Session
s = Session()
print(
"Spam Call by SkyFish - BODOAMAT\nThis tool delays 5 seconds per spam so as not to limit!\nUse Country Code (ex: 62xxxxx29)"
)
try:
no = int(input("No : "))
jml = int(input("Count : "))
print()
except:
print("\n\t* Only Number *")
sys.exit()
url = "https://www.citcall.com/demo/misscallapi.php"
tkn = s.get(url).text
token = re.findall(r'id="csrf_token" value="(.*?)">', tkn)[0]
headers = {'x-requested-with': 'XMLHttpRequest'}
data = {'cid': no, 'trying': '0', 'csrf_token': token}
n = 0
try:
while n < jml:
send = s.post(url, data=data, headers=headers).text
time.sleep(4.8)
def initClient(self):
session = Session()
session.auth = HTTPBasicAuth(self.username, self.password)
self.client = Client(self.endpoint,
transport=Transport(session=session))
