def __init__(self, client_id=None, secret=None):
"""
知乎客户端，这是获取所有类的入口。
:param str|unicode client_id: 客户端 ID。
:param str|unicode secret: 客户端 ID 对应的 SECRET KEY。
:rtype: :class:`.ZhihuClient`
"""
self._session = requests.session()
# remove SSL Verify
self._session.verify = False
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# Add auto retry for session
self._session.mount('http://', ADAPTER_WITH_RETRY)
self._session.mount('https://', ADAPTER_WITH_RETRY)
# client_id and secret shouldn't have default value
# after zhihu open api
self._client_id = client_id or CLIENT_ID
self._secret = secret or APP_SECRET
self._login_auth = BeforeLoginAuth(self._client_id)
self._token = None
def leancloud_init():
'''
leancloud初始化
:return:
'''
urllib3.disable_warnings()
leancloud.init('lXyQBue2G2I80NX9OIFY7TRk', 'NkLOGPRHeVrFdJOQiDIGVGJ7')
def configure_logging(log_config=None, debug=None, forward_stdout=None,
hide_ssl_warnings=None):
"""Configure the logging
Loading logging configuration file which is defined in the general
configuration file and configure the logging system.
Setting the level of console handler to DEBUG mode if debug option is set
as True.
Wrap the stdout stream by StdoutLogger.
"""
if log_config is None:
log_config = CONF.migrate.log_config
if debug is None:
debug = CONF.migrate.debug
if forward_stdout is None:
forward_stdout = CONF.migrate.forward_stdout
if hide_ssl_warnings is None:
hide_ssl_warnings = CONF.migrate.hide_ssl_warnings
with open(log_config, 'r') as f:
config.dictConfig(yaml.load(f))
if debug:
logger = logging.getLogger('cloudferry')
for handler in logger.handlers:
if handler.name == 'console':
handler.setLevel(logging.DEBUG)
if forward_stdout:
sys.stdout = StdoutLogger()
if hide_ssl_warnings:
urllib3.disable_warnings()
def __init__(self, config):
"""
Initialization is responsible for fetching service instance objects for each vCenter instance
pyvmomi has some security checks enabled by default from python 2.7.9 onward to connect to vCenter.
"""
# Holds all the VMs' instanceuuid that are discovered for each of the vCenter. Going ahead it would hold all the
# other managed objects of vCenter that would be monitored.
self.mors = {} # Now mars is act as <key,value>. here key is instance UUID and Values is morf Id
self.params = config
global metrics
global counters
metrics = util.parse_metrics()
counters = util.parse_counters()
self.needed_metrics = {}
self.configured_metrics = {}
self.refresh_rates = {}
self.service_instance = ""
for k, v in metrics.items():
self.configured_metrics.update({util.get_counter(k): v})
if sys.version_info > (2, 7, 9) and sys.version_info < (3, 0, 0):
# https://www.python.org/dev/peps/pep-0476/
# Look for 'Opting out' section in this that talks about disabling the certificate verification
# Following line helps to disable globally
ssl._create_default_https_context = ssl._create_unverified_context
# Disabling the security warning message, as the certificate verification is disabled
urllib3.disable_warnings()
try:
service_instance = connect.SmartConnectNoSSL(host=self.params['host'],
user=self.params['username'],
pwd=self.params['password'],
port=int(self.params['port']))
util.sendEvent("Plugin vmware", "Sucessfully connected to vCenter: [" + self.params['host'] + "]", "info")
atexit.register(connect.Disconnect, service_instance)
self.service_instance = service_instance
self._cache_metrics_metadata(self.params['host'])
except KeyError as ke:
util.sendEvent("Plugin vmware: Key Error", "Improper param.json, key missing: [" + str(ke) + "]", "error")
# sys.exit(-1)
except ConnectionError as ce:
util.sendEvent("Plugin vmware: Error connecting to vCenter",
"Could not connect to the specified vCenter host: [" + str(ce) + "]", "critical")
except Exception as se:
util.sendEvent("Plugin vmware: Unknown Error", "[" + str(se) + "]", "critical")
# sys.exit(-1)
except vim.fault.InvalidLogin as il:
util.sendEvent("Plugin vmware: Error logging into vCenter",
"Could not login to the specified vCenter host: [" + str(il) + "]", "critical")
def main(filename):
disable_warnings()
if not os.path.isfile(os.path.realpath(filename)):
sys.stderr.write("File %s not found.\n" % filename)
sys.exit(1)
basename = os.path.splitext(filename)[0]
response = get_subtitleinfo(filename)
sys.stdout.write("Requesting subtitle file...\n")
subtitles = set([])
for count in xrange(len(response.json())):
if count != 0:
_basename = "%s-alt.%s" % (basename, count)
else:
_basename = "%s.%s" % (basename, count)
for fileinfo in response.json()[count]['Files']:
url = fileinfo['Link']
ext = fileinfo['Ext']
_response = requests.get(url, verify=False)
filename = "%s.%s" % (_basename, ext)
if _response.ok and _response.text not in subtitles:
subtitles.add(_response.text)
fobj = open(filename, 'w')
fobj.write(_response.text.encode("UTF8"))
fobj.close()
def __init__(self, api_key):
self.api_key = base64.encodestring('%s:' % api_key).replace('\n', '')
self.user_agent = "SalesMonitoringAgent"
self.headers = {'Authorization': "Basic %s" % self.api_key }
self.headers['Content-type'] = 'application/json'
self.headers['Accept'] = 'application/json'
urllib3.disable_warnings()
def __init__(self):
urllib3.disable_warnings()
self.header = {'HOST': 'www.zhihu.com', 'Referer': 'https://www.zhihu.com/people/ipreacher/answers',
'USER-AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/55.0.2883.95 Safari/537.36'}
self.session = requests.session()
self.session.get('https://www.zhihu.com/', headers=self.header, verify=False)
def validate(self):
"""Run validation using HTTP requests against validation host
Using rules provided by spec, perform requests against validation host
for each rule. Request response is verified to match the spec respsonse
rule. This will yield either a :py:cls:`ValidationPass` or
:py:cls:`ValidationFail` response.
"""
session = Session()
if not self.verify and hasattr(urllib3, 'disable_warnings'):
urllib3.disable_warnings()
for rule in self.spec.get_rules():
req = rule.get_request(self.host, self.port)
if self.debug:
pprint.pprint(req.__dict__)
try:
resp = session.send(req.prepare(), allow_redirects=False,
verify=self.verify)
if self.debug:
pprint.pprint(resp.__dict__)
if rule.matches(resp):
yield ValidationPass(rule=rule, request=req, response=resp)
except (ConnectionError, SSLError) as exc:
# No response yet
yield ValidationFail(rule=rule, request=req, response=None,
error=exc)
except ValidationError as exc:
# Response received, validation error
yield ValidationFail(rule=rule, request=req, response=resp,
error=exc)
def main():
from requests.packages import urllib3
urllib3.disable_warnings()
parser = build_parser()
args = parser.parse_args()
plugins = []
try:
for plugin_name in args.plugins:
cl = import_class(plugin_name)
plugins.append(cl())
nocmd = None
if args.nocmd is not None:
cl = import_class(args.nocmd)
nocmd = cl()
except Exception as e:
parser.error(e.message)
tg = TGBot(args.token, plugins=plugins, no_command=nocmd, db_url=args.db_url)
if args.list:
tg.print_commands()
return
if args.token is None:
parser.error('--token is required')
if args.webhook is None:
tg.run(polling_time=args.polling)
else:
tg.run_web(args.webhook[0], host='0.0.0.0', port=int(args.webhook[1]))
def patch_urllib3():
try:
# disable the annoying AppenginePlatformWarning's
import urllib3
urllib3.disable_warnings()
except ImportError:
pass
def configure_logging(debug=None, verbose=None):
"""Configure the logging system
Load the logging configuration file and configure the
logging system.
:param debug: allows to enable/disable the console
debug mode
:param verbose: allows to verbose the debug messages
"""
with open(CONF.basic.log_config, 'r') as f:
logging.config.dictConfig(yaml.load(f))
logger = logging.getLogger()
if not debug:
for handler in logger.handlers:
if handler.name == 'console':
handler.setLevel(logging.INFO)
handler.setFormatter(logging.Formatter())
break
if verbose:
logger.setLevel(logging.DEBUG)
if CONF.basic.hide_ssl_warnings:
urllib3.disable_warnings()
CONF.log_opt_values(logging, logging.DEBUG)
def __init__(self, address, **kwargs):
"""
explain me ; )
"""
utils.validate_uri(address)
# TODO: update me with a config file entry need to update to get the version from docker and use that.
self.API_VERSION = DOCKER_API_VERSION
self.url = urlparse(address)
if self.url.scheme == 'https':
# TODO: Need to allow for ca to be passed if not disable warnings.
urllib3.disable_warnings()
for cert_name_type in ('ca', 'cert', 'key'):
cert_path = utils.validate_path(os.path.join(kwargs['ssl_cert_path'], "{0}.pem".format(cert_name_type))) \
if 'ssl_cert_path' in kwargs and kwargs['ssl_cert_path'] else None
setattr(self, 'SSL_{0}_PATH'.format(cert_name_type.upper()), cert_path)
self.SSL_VERIFY = kwargs['verify'] if 'verify' in kwargs and isinstance(kwargs['verify'], bool) else True
if not self.SSL_VERIFY:
self.SSL_CA_PATH = None
client_certs = (self.SSL_CERT_PATH, self.SSL_KEY_PATH) if self.SSL_KEY_PATH and self.SSL_CERT_PATH else None
tls_config = docker.tls.TLSConfig(client_cert=client_certs, ca_cert=self.SSL_CA_PATH, verify=self.SSL_VERIFY)
self._client_session = docker.Client(self.url.geturl(), tls=tls_config, timeout=DOCKER_DEFAULT_TIMEOUT, version=self.API_VERSION)
else:
self._client_session = docker.Client(self.url.geturl(), timeout=DOCKER_DEFAULT_TIMEOUT, version=self.API_VERSION)
self._docker_info = self._client_session.version()
self._injector = None
def disable_warnings(cls, debug):
failures = 0
exmsg = ''
try:
import requests.packages.urllib3 as ul3
if debug:
pdbg("Using requests.packages.urllib3 to disable warnings")
#ul3.disable_warnings(ul3.exceptions.InsecureRequestWarning)
#ul3.disable_warnings(ul3.exceptions.InsecurePlatformWarning)
ul3.disable_warnings()
except Exception as ex:
failures += 1
exmsg += formatex(ex) + '-' * 64 + '\n'
# i don't know why under Ubuntu, 'pip install requests'
# doesn't install the requests.packages.* packages
try:
import urllib3 as ul3
if debug:
pdbg("Using urllib3 to disable warnings")
ul3.disable_warnings()
except Exception as ex:
failures += 1
exmsg += formatex(ex)
if failures >= 2:
perr("Failed to disable warnings for Urllib3.\n"
"Possibly the requests library is out of date?\n"
"You can upgrade it by running '{}'.\nExceptions:\n{}".format(
const.PipUpgradeCommand, exmsg))
def sender(self):
try:
from requests.packages.urllib3 import disable_warnings
disable_warnings()
except:
logger.info('Unable to disable https warnings. Expect some spam if using https nzb providers.')
try:
logger.info('parameters set to %s' % self.params)
logger.info('sending now to %s' % self.sab_url)
sendit = requests.post(self.sab_url, data=self.params, verify=False)
except:
logger.info('Failed to send to client.')
return {'status': False}
else:
sendresponse = sendit.json()
logger.info(sendresponse)
if sendresponse['status'] is True:
queue_params = {'status': True,
'nzo_id': ''.join(sendresponse['nzo_ids']),
'queue': {'mode': 'queue',
'search': ''.join(sendresponse['nzo_ids']),
'output': 'json',
'apikey': mylar.CONFIG.SAB_APIKEY}}
else:
queue_params = {'status': False}
return queue_params
def _get_pool_manager(verify, cert_file, key_file):
global _pool_manager
default_pool_args = dict(maxsize=32,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=_default_certs,
headers=_default_headers,
timeout=_default_timeout)
if cert_file is None and verify is None and 'DX_CA_CERT' not in os.environ:
with _pool_mutex:
if _pool_manager is None:
if 'HTTPS_PROXY' in os.environ:
proxy_params = _get_proxy_info(os.environ['HTTPS_PROXY'])
default_pool_args.update(proxy_params)
_pool_manager = urllib3.ProxyManager(**default_pool_args)
else:
_pool_manager = urllib3.PoolManager(**default_pool_args)
return _pool_manager
else:
# This is the uncommon case, normally, we want to cache the pool
# manager.
pool_args = dict(default_pool_args,
cert_file=cert_file,
key_file=key_file,
ca_certs=verify or os.environ.get('DX_CA_CERT') or requests.certs.where())
if verify is False or os.environ.get('DX_CA_CERT') == 'NOVERIFY':
pool_args.update(cert_reqs=ssl.CERT_NONE, ca_certs=None)
urllib3.disable_warnings()
if 'HTTPS_PROXY' in os.environ:
proxy_params = _get_proxy_info(os.environ['HTTPS_PROXY'])
pool_args.update(proxy_params)
return urllib3.ProxyManager(**pool_args)
else:
return urllib3.PoolManager(**pool_args)
def download(self):
"""
download the NXDL definitions described by ``ref``
"""
_msg = u'disabling warnings about GitHub self-signed https certificates'
#requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
disable_warnings(InsecureRequestWarning)
creds = get_BasicAuth_credentials()
content = None
for _retry in range(GITHUB_RETRY_COUNT): # noqa
try:
if creds is None:
content = requests.get(self.zip_url, verify=False)
else:
content = requests.get(self.zip_url,
auth=(creds['user'], creds['password']),
verify=False,
)
except requests.exceptions.ConnectionError as _exc:
_msg = 'ConnectionError from ' + self.zip_url
_msg += '\n' + str(_exc)
raise IOError(_msg)
else:
break
return content
def requests_get_mock(*args, **kwargs):
self.assertEquals(kwargs.get('timeout'), 5)
with requests.sessions.Session() as session:
from requests.packages import urllib3
urllib3.disable_warnings()
return requests_mock.create_response(
session.request('GET', args[0]), json={})
def _suppress_urllib3_error(self):
try:
# Try to suppress user warnings
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from requests.packages.urllib3 import disable_warnings
disable_warnings(InsecureRequestWarning)
except:
pass
def init(self, url=None, username=None, password=None):
self.url = url
self.username = username
self.password = password
self.auth = HTTPBasicAuth(username, password)
self.content_type = {'Content-Type': 'application/xml'}
self.accept_header = {'Accept': 'application/xml'}
disable_warnings()
def main():
urllib3.disable_warnings()
path, url = sys.argv[1:]
with open(os.path.expanduser(path)) as auth_source:
username, password = auth_source.read().strip().split(':')
response = requests.get(url, auth=HTTPBasicAuth(username, password))
if response.status_code != 200 or not response.text.startswith('<SUCCESS CODE="20'):
print response.text
def __init__(self):
urllib3.disable_warnings()
self.conn = mysql.connector.connect(user='******', password='', database='f1000')
self.cursor = self.conn.cursor(buffered=True)
self.header = {'HOST': 'f1000.com', 'Origin': 'https://f1000.com',
'Referer': 'https://f1000.com/prime/recommendations',
'USER-AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/55.0.2883.95 Safari/537.36'}
self.session = requests.session()
def main():
args = get_args()
urllib3.disable_warnings()
si = None
context = None
if hasattr(ssl, 'SSLContext'):
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.verify_mode = ssl.CERT_NONE
if context:
# Python >= 2.7.9
si = SmartConnect(host=args.host,
port=int(args.port),
user=args.user,
pwd=args.password,
sslContext=context)
else:
# Python >= 2.7.7
si = SmartConnect(host=args.host,
port=int(args.port),
user=args.user,
pwd=args.password)
atexit.register(Disconnect, si)
print "Connected to vCenter Server"
content = si.RetrieveContent()
datacenter = get_obj(content, [vim.Datacenter], args.datacenter_name)
if not datacenter:
raise Exception("Couldn't find the Datacenter with the provided name "
"'{}'".format(args.datacenter_name))
cluster = get_obj(content, [vim.ClusterComputeResource], args.cluster_name,
datacenter.hostFolder)
if not cluster:
raise Exception("Couldn't find the Cluster with the provided name "
"'{}'".format(args.cluster_name))
host_obj = None
for host in cluster.host:
if host.name == args.host_name:
host_obj = host
break
vm_folder = datacenter.vmFolder
template = get_obj(content, [vim.VirtualMachine], args.template_name,
vm_folder)
if not template:
raise Exception("Couldn't find the template with the provided name "
"'{}'".format(args.template_name))
location = _get_relocation_spec(host_obj, cluster.resourcePool)
_take_template_snapshot(si, template)
_clone_vm(si, template, args.vm_name, vm_folder, location)
def __init__(self, config=Config()):
""":type _config: :class:`Config`"""
self._config = config
self.session_id = None
self.user = None
self.country_code = 'US' # Enable Trial Mode
self.client_unique_key = None
try:
urllib3.disable_warnings() # Disable OpenSSL Warnings in URLLIB3
except:
pass
def inner(insecure, **kwargs):
if insecure:
# If user chose to use insecure explicitly, ignore warnings...
try:
import requests.packages.urllib3 as urllib3
urllib3.disable_warnings()
click.echo("Warning: Not verifying TLS certificates.")
except ImportError:
pass # If requests moves urllib3 around
appcheck = get_appcheck(insecure=insecure)
f(appcheck, **kwargs)
def main():
urllib3.disable_warnings()
parser = argparse.ArgumentParser(description='Process user input for piazza queries')
parser.add_argument('-q', '--query', nargs="+")
parser.add_argument('-t', '--tag', nargs=1)
#parser.add_argument('-r', '--range', nargs=2)
parser.add_argument('-i', '--instructor-only', action='store_true')
parser.add_argument('-p', '--pinned', action='store_true')
parser.add_argument('-f', '--following', action='store_true')
parser.add_argument('-l', '--force-login', action='store_true')
args = parser.parse_args()
queryObj = QueryObj()
queryObj.add_query(args.query)
queryObj.add_tag(args.tag)
#queryObj.add_time_range(args.range)
queryObj.bool_inst_notes(args.instructor_only)
queryObj.bool_pinned(args.pinned)
queryObj.bool_following(args.following)
loginfile = os.path.expanduser("~") + "/.pizza"
if not args.force_login:
try:
pkl = pickle.load(open(loginfile,"rb"))
data = {'email': pkl['email'], 'password': pkl['password'].decode('rot13')}
except IOError:
email = raw_input('Piazza Email: ')
password = getpass.getpass()
data = {'email': email, 'password': password}
pkl = {'email': email, 'password': password.encode('rot13')}
pickle.dump(pkl, open(loginfile, "wb"))
piazza = Piazza()
piazza.user_login(data['email'], data['password'])
user_status = piazza.get_user_status()
classes = user_status['networks']
classes = sorted(classes, key=lambda k: k['status'])
# list classes
print("Choose a Class")
counter = 1
for c in classes:
info = c['name']
if c['status'] == 'inactive':
info = '(inactive) ' + info
print '{0:2d}: {1:s}'.format(counter, info)
counter = counter + 1
index = raw_input('Class Number: ')
network = piazza.network(classes[int(index) - 1]['id'])
feed_processor = FeedProcessor(network, queryObj)
curses.wrapper(summary_viewer.view_summaries, feed_processor, network)
def __init__(self):
super(HttpMixin, self).__init__()
self._http_log = logger
if not self.verify:
if self._http_log.handlers:
self._http_log.warn(HTTP_INSECURE_MESSAGE)
else:
print(HTTP_INSECURE_MESSAGE)
from requests.packages.urllib3 import disable_warnings
disable_warnings()
def configure_urllib3():
# Disable verbose urllib3 warnings and log messages
urllib3.disable_warnings(category=urllib3.exceptions.InsecurePlatformWarning)
logging.getLogger('dxpy.packages.requests.packages.urllib3.connectionpool').setLevel(logging.ERROR)
# Trust DNAnexus S3 upload tunnel
def _match_hostname(cert, hostname):
if hostname == "ul.cn.dnanexus.com":
hostname = "s3.amazonaws.com"
match_hostname(cert, hostname)
urllib3.connection.match_hostname = _match_hostname
def disable_ssl_warnings():
try:
from requests.packages import urllib3
urllib3.disable_warnings()
except ImportError:
pass
try:
import urllib3
urllib3.disable_warnings()
except ImportError:
pass
def __init__(self):
"""
Creates a twitter endpoint that can
send authenticated requests.
"""
self.auth = requests_oauthlib.OAuth1(CLIENT_KEY,
client_secret=CLIENT_SECRET,
resource_owner_key=RESOURCE_OWNER_KEY,
resource_owner_secret=RESOURCE_OWNER_SECRET)
self.response = None
self.json = None
# suppress urllib3 InsecurePlatformWarning
urllib3.disable_warnings()
def on_task_start(self, task, config):
if config is False:
task.requests.verify = False
# Disabling verification results in a warning for every HTTPS
# request:
# "InsecureRequestWarning: Unverified HTTPS request is being made.
# Adding certificate verification is strongly advised. See:
# https://urllib3.readthedocs.io/en/latest/security.html"
# Disable those warnings because the user has explicitly disabled
# verification and the warning is not beneficial.
# This change is permanent rather than task scoped, but there won't
# be any warnings to disable when verification is enabled.
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def __init__(self,
fileconf=None,
client_token=None,
cluster_url=None,
cluster_unsafe=False,
cluster_timeout=None,
storage_url=None,
storage_unsafe=False,
retry_count=5,
retry_wait=1.0,
cluster_custom_certificate=None,
storage_custom_certificate=None,
sanitize_bucket_paths=True,
show_bucket_warnings=True):
"""Create a connection to a cluster with given config file, options or environment variables.
Available environment variable are
`QARNOT_CLUSTER_URL`, `QARNOT_CLUSTER_UNSAFE`, `QARNOT_CLUSTER_TIMEOUT` and `QARNOT_CLIENT_TOKEN`.
:param fileconf: path to a qarnot configuration file or a corresponding dict
:type fileconf: str or dict
:param str client_token: API Token
:param str cluster_url: (optional) Cluster url.
:param bool cluster_unsafe: (optional) Disable certificate check
:param int cluster_timeout: (optional) Timeout value for every request
:param str storage_url: (optional) Storage service url.
:param bool storage_unsafe: (optional) Disable certificate check
:param int retry_count: (optional) ConnectionError retry count. Default to 5.
:param float retry_wait: (optional) Retry on error wait time, progressive. (wait * (retry_count - retry_num). Default to 1s
:param bool sanitize_bucket_paths: (optional) Flag to automatically sanitize bucket paths (remove extra slashes). Default to true
:param bool show_bucket_warnings: (optional) Flag to show warnings of bucket paths sanitization. Default to true
Configuration sample:
.. code-block:: ini
[cluster]
# url of the REST API
url=https://localhost
# No SSL verification ?
unsafe=False
[client]
# auth string of the client
token=login
[storage]
url=https://storage
unsafe=False
"""
self._version = "qarnot-sdk-python/" + __version__
self._http = requests.session()
self._retry_count = retry_count
self._retry_wait = retry_wait
self._sanitize_bucket_paths = sanitize_bucket_paths
self._show_bucket_warnings = show_bucket_warnings
if fileconf is not None:
self.storage = None
if isinstance(fileconf, dict):
warnings.warn(
"Dict config should be replaced by constructor explicit arguments."
)
self.cluster = None
if fileconf.get('cluster_url'):
self.cluster = fileconf.get('cluster_url')
if fileconf.get('storage_url'):
self.storage = fileconf.get('storage_url')
auth = fileconf.get('client_auth')
self.timeout: int = int(fileconf.get('cluster_timeout'))
if fileconf.get('cluster_unsafe'):
self._http.verify = False
elif fileconf.get('cluster_custom_certificate'):
self._http.verify = fileconf.get(
'cluster_custom_certificate')
else:
cfg = config.ConfigParser()
with open(fileconf) as cfg_file:
cfg.read_string(cfg_file.read())
self.cluster = None
if cfg.has_option('cluster', 'url'):
self.cluster = cfg.get('cluster', 'url')
if cfg.has_option('storage', 'url'):
self.storage = cfg.get('storage', 'url')
if cfg.has_option('client', 'token'):
auth = cfg.get('client', 'token')
elif cfg.has_option('client', 'auth'):
warnings.warn('auth is deprecated, use token instead.')
auth = cfg.get('client', 'auth')
else:
auth = None
self.timeout = None
if cfg.has_option('cluster', 'timeout'):
self.timeout = cfg.getint('cluster', 'timeout')
if cfg.has_option('cluster', 'unsafe') \
and cfg.getboolean('cluster', 'unsafe'):
self._http.verify = False
elif cfg.has_option('cluster', 'custom_certificate'):
self._http.verify = cfg.get('cluster',
'custom_certificate')
if cfg.has_option('storage', 'unsafe') \
and cfg.getboolean('storage', 'unsafe'):
storage_unsafe = True
if cfg.has_option('storage', 'custom_certificate'):
storage_custom_certificate = cfg.get(
'storage', 'custom_certificate')
else:
self.cluster = cluster_url
self.timeout = cluster_timeout
self._http.verify = not cluster_unsafe
if not cluster_unsafe and cluster_custom_certificate:
self._http.verify = cluster_custom_certificate
self.storage = storage_url
auth = client_token
if not self._http.verify:
urllib3.disable_warnings()
if self.cluster is None:
self.cluster = os.getenv("QARNOT_CLUSTER_URL")
if self.storage is None:
self.storage = os.getenv("QARNOT_STORAGE_URL")
if auth is None:
auth = os.getenv("QARNOT_CLIENT_TOKEN")
if os.getenv("QARNOT_CLUSTER_UNSAFE") is not None:
self._http.verify = not os.getenv("QARNOT_CLUSTER_UNSAFE") in [
"true", "True", "1"
]
if os.getenv("QARNOT_CLUSTER_TIMEOUT") is not None:
self.timeout = int(os.getenv("QARNOT_CLUSTER_TIMEOUT"))
if auth is None:
raise QarnotGenericException("Token is mandatory.")
self._http.headers.update({"Authorization": auth})
self._http.headers.update({"User-Agent": self._version})
if self.cluster is None:
self.cluster = "https://api.qarnot.com"
api_settings = self._get(get_url("settings")).json()
if self.storage is None:
self.storage = api_settings.get("storage",
"https://storage.qarnot.com")
if self.storage is None: # api_settings["storage"] is None
self._s3client = None
self._s3resource = None
return
user = self.user_info
session = boto3.session.Session()
conf = botocore.config.Config(user_agent=self._version)
should_verify_or_certificate_path = True
if storage_unsafe:
should_verify_or_certificate_path = not storage_unsafe
elif storage_custom_certificate is not None:
should_verify_or_certificate_path = storage_custom_certificate
self._s3client = session.client(
service_name='s3',
aws_access_key_id=user.email,
aws_secret_access_key=auth,
verify=should_verify_or_certificate_path,
endpoint_url=self.storage,
config=conf)
self._s3resource = session.resource(
service_name='s3',
aws_access_key_id=user.email,
aws_secret_access_key=auth,
verify=should_verify_or_certificate_path,
endpoint_url=self.storage,
config=conf)
def main():
# 去除 warning
urllib3.disable_warnings()
url = 'https://finland.fi/category/facts-stats-and-info/'
fun_call(url)
def request(self, host, handler, request_body, verbose=0):
"""Replace the xmlrpc request function.
Process xmlrpc request via requests library.
Args:
host: Target host
handler: Target PRC handler.
request_body: XML-RPC request body.
verbose: Debugging flag.
Returns:
Parsed response.
Raises:
RequestException: Error in requests
"""
if verbose:
self._debug()
if not self._check_ssl_cert:
disable_warnings()
headers = {'User-Agent': self.user_agent, 'Content-Type': 'text/xml', }
# Need to be done because the schema(http or https) is lost in
# xmlrpc.Transport's init.
if self._use_https:
url = "https://{host}/{handler}".format(host=host, handler=handler)
else:
url = "http://{host}/{handler}".format(host=host, handler=handler)
# TODO Construct kwargs query instead
try:
if self._authtype == "basic":
response = requests.post(
url,
data=request_body,
headers=headers,
verify=self._check_ssl_cert,
auth=HTTPBasicAuth(
self._username, self._password),
proxies=self._proxies)
elif self._authtype == "digest":
response = requests.post(
url,
data=request_body,
headers=headers,
verify=self._check_ssl_cert,
auth=HTTPDigestAuth(
self._username, self._password),
proxies=self._proxies)
else:
response = requests.post(
url,
data=request_body,
headers=headers,
verify=self._check_ssl_cert,
proxies=self._proxies)
response.raise_for_status()
except RequestException as error:
raise xmlrpc_client.ProtocolError(url,
error.message,
traceback.format_exc(),
response.headers)
return self.parse_response(response)
def runCouchPotato(options,
base_path,
args,
data_dir=None,
log_dir=None,
Env=None,
desktop=None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = sp(os.path.join(data_dir, 'database'))
old_db_path = os.path.join(data_dir, 'couchpotato.db')
# Remove database folder if both exists
if os.path.isdir(db_path) and os.path.isfile(old_db_path):
db = SuperThreadSafeDatabase(db_path)
db.open()
db.destroy()
# Check if database exists
db = SuperThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = sp(os.path.join(data_dir, 'db_backup'))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
# Only consider files being a direct child of the backup_path
if root == backup_path:
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
try:
os.remove(os.path.join(root, backup_file))
except:
pass
else:
existing_backups.append((int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = sp(
os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
zipf = tarfile.open(new_backup, 'w:gz')
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename),
arcname='database/%s' %
os.path.join(root[len(db_path) + 1:], zfilename))
zipf.close()
# Open last
db.open()
else:
db.create()
# Force creation of cachedir
log_dir = sp(log_dir)
cache_dir = sp(os.path.join(data_dir, 'cache'))
python_cache = sp(os.path.join(cache_dir, 'python'))
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not os.path.exists(python_cache):
os.mkdir(python_cache)
session = requests.Session()
session.max_redirects = 5
# Register environment settings
Env.set('app_dir', sp(base_path))
Env.set('data_dir', sp(data_dir))
Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db', db)
Env.set('http_opener', session)
Env.set('cache_dir', cache_dir)
Env.set('cache', FileSystemCache(python_cache))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default=False, type='bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default=False, type='bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in [
'enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado',
'requests'
]:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Disable SSL warning
disable_warnings()
# Use reloader
reloader = debug is True and development and not Env.get(
'desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
'%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or
options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'),
'a',
500000,
10,
encoding=Env.get('encoding'))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
# Check available space
try:
total_space, available_space = getFreeSpace(data_dir)
if available_space < 100:
log.error(
'Shutting down as CP needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left',
available_space)
return
except:
log.error('Failed getting diskspace: %s', traceback.format_exc())
def customwarn(message, category, filename, lineno, file=None, line=None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') +
'/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
if not api_key:
api_key = uuid4().hex
Env.setting('api_key', value=api_key)
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default='0.0.0.0')
host6 = Env.setting('host6', default='::')
# app.debug = development
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default=5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'host6': host6 if host6 and len(host6) > 0 else '::',
'ssl_cert': Env.setting('ssl_cert', default=None),
'ssl_key': Env.setting('ssl_key', default=None),
}
# Load the app
application = Application(
[],
log_function=lambda x: None,
debug=config['use_reloader'],
gzip=True,
cookie_secret=api_key,
login_url='%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(
".*$",
[
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {
"url": web_base + 'docs/'
}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(
".*$",
[('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {
'path':
sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))
})])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root=sp(base_path))
loader.run()
# Fill database with needed stuff
fireEvent('database.setup')
if not db_exists:
fireEvent('app.initialize', in_order=True)
fireEvent('app.migrate')
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def reload_hook():
fireEvent('app.shutdown')
add_reload_hook(reload_hook)
# Some logging and fire load event
try:
log.info('Starting server on port %(port)s', config)
except:
pass
fireEventAsync('app.load')
ssl_options = None
if config['ssl_cert'] and config['ssl_key']:
ssl_options = {
'certfile': config['ssl_cert'],
'keyfile': config['ssl_key'],
}
server = HTTPServer(application,
no_keep_alive=True,
ssl_options=ssl_options)
try_restart = True
restart_tries = 5
while try_restart:
try:
server.listen(config['port'], config['host'])
server.listen(config['port'], config['host6'])
loop.start()
server.close_all_connections()
server.stop()
loop.close(all_fds=True)
except Exception as e:
log.error('Failed starting: %s', traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info(
'Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds',
(config.get('port'), restart_tries))
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except ValueError:
return
except:
pass
raise
try_restart = False
def get_lastest_history_passage(query):
# query为要爬取的公众号名称
# 公众号主页
url = 'https://mp.weixin.qq.com'
# 设置headers
header = {
"HOST": "mp.weixin.qq.com",
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3239.132"
}
from requests.packages import urllib3
urllib3.disable_warnings() # 关闭警告
# 读取登陆成功后获取到的cookies
with open(COOKIES_PATH, 'r', encoding='utf-8') as f:
cookie = f.read()
cookies = json.loads(cookie)
print("Cookies loading successfully.")
# 增加重试连接次数
session = requests.Session()
session.keep_alive = False
# 增加重试连接次数
session.adapters.DEFAULT_RETRIES = 511
try:
# 登录之后的微信公众号首页url变化为：https://mp.weixin.qq.com/cgi-bin/home?t=home/index&lang=zh_CN&token=1849751598，从这里获取token信息
response = session.get(url=url, cookies=cookies, verify=False)
token = re.findall(r'token=(\d+)', str(response.url))[0]
time.sleep(15)
# 搜索微信公众号的接口地址
search_url = 'https://mp.weixin.qq.com/cgi-bin/searchbiz?'
# 搜索微信公众号接口需要传入的参数，有三个变量：微信公众号token、随机数random、搜索的微信公众号名字
query_id = {
'action': 'search_biz',
'token': token,
'lang': 'zh_CN',
'f': 'json',
'ajax': '1',
'random': random.random(),
'query': query,
'begin': '0',
'count': '5'
}
# 打开搜索微信公众号接口地址，需要传入相关参数信息如：cookies、params、headers
search_response = session.get(
search_url,
cookies=cookies,
headers=header,
params=query_id)
# 取搜索结果中的第一个公众号
lists = search_response.json().get('list')
logging.info("查询到公众号信息列表：" + str(lists))
except Exception as e:
print(str(e))
logger.error(str(e))
try:
# 获取这个公众号的fakeid，后面爬取公众号文章需要此字段
fakeid = lists[0].get('fakeid')
# 微信公众号文章接口地址
appmsg_url = 'https://mp.weixin.qq.com/cgi-bin/appmsg?'
# 搜索文章需要传入几个参数：登录的公众号token、要爬取文章的公众号fakeid、随机数random
query_id_data = {
'token': token,
'lang': 'zh_CN',
'f': 'json',
'ajax': '1',
'random': random.random(),
'action': 'list_ex',
'begin': '0', # 不同页，此参数变化，变化规则为每页加5
'count': '5',
'query': '',
'fakeid': fakeid,
'type': '9'
}
# 打开搜索的微信公众号文章列表页
appmsg_response = session.get(
appmsg_url,
cookies=cookies,
headers=header,
params=query_id_data,
)
# 获取文章总数
# max_num = appmsg_response.json().get('app_msg_cnt')
# 只爬取最新
# max_num = 10
# 每页至少有5条，获取文章总的页数，爬取时需要分页爬
# num = int(int(max_num) / 5)
num = 2
# 起始页begin参数，往后每页加5
begin = 0
# 获取公众号在MySQL数据库中的ID
official_account_id = db_core.search_wechat_account_is_existed(query)
# 开始爬取
print("开始爬取!")
while num + 1 > 0:
query_id_data = {
'token': token,
'lang': 'zh_CN',
'f': 'json',
'ajax': '1',
'random': random.random(),
'action': 'list_ex',
'begin': '{}'.format(str(begin)),
'count': '5',
'query': '',
'fakeid': fakeid,
'type': '9'
}
# print('正在翻页：--------------', begin)
time.sleep(15)
# 获取每一页文章的标题和链接地址，并写入本地文本中
query_fakeid_response = requests.get(
appmsg_url,
cookies=cookies,
headers=header,
params=query_id_data,
timeout=3)
fakeid_list = query_fakeid_response.json().get('app_msg_list')
if fakeid_list:
for item in fakeid_list:
content_link = item.get('link') # 文章链接
content_title = item.get('title') # 文章标题
content_create_time = item.get('create_time') # 文章发布时间
content_update_time = item.get('update_time') # 文章更新时间
# 组装MongoDB插入数据
docs = {"passage_title": content_title,
"official_account_id": official_account_id,
"passage_link":content_link,
"passage_create_time":time_format(content_create_time),
"passage_update_time":time_format(content_update_time)}
docs_mongo = {"$set":docs}
db_core.insert_account_passage_link(content_title, content_link, official_account_id) # MySQL数据库更新
docs_db_core.insert_docs(docs_mongo, passage_link=content_link) # MongoDB数据库更新
num -= 1
begin = int(begin)
begin += 5
print("完成爬取.")
except Exception as e:
print(str(e))
logger.error(str(e))
finally:
print("爬虫进程结束.")
def DXHTTPRequest(resource,
data,
method='POST',
headers=None,
auth=True,
timeout=DEFAULT_TIMEOUT,
use_compression=None,
jsonify_data=True,
want_full_response=False,
decode_response_body=True,
prepend_srv=True,
session_handler=None,
max_retries=DEFAULT_RETRIES,
always_retry=False,
**kwargs):
'''
:param resource: API server route, e.g. "/record/new". If *prepend_srv* is False, a fully qualified URL is expected. If this argument is a callable, it will be called just before each request attempt, and expected to return a tuple (URL, headers). Headers returned by the callback are updated with *headers* (including headers set by this method).
:type resource: string
:param data: Content of the request body
:type data: list or dict, if *jsonify_data* is True; or string or file-like object, otherwise
:param headers: Names and values of HTTP headers to submit with the request (in addition to those needed for authentication, compression, or other options specified with the call).
:type headers: dict
:param auth: Overrides the *auth* value to pass through to :meth:`requests.request`. By default a token is obtained from the ``DX_SECURITY_CONTEXT``.
:type auth: tuple, object, True (default), or None
:param timeout: HTTP request timeout, in seconds
:type timeout: float
:param config: *config* value to pass through to :meth:`requests.request`
:type config: dict
:param use_compression: "snappy" to use Snappy compression, or None
:type use_compression: string or None
:param jsonify_data: If True, *data* is converted from a Python list or dict to a JSON string
:type jsonify_data: boolean
:param want_full_response: If True, the full :class:`requests.Response` object is returned (otherwise, only the content of the response body is returned)
:type want_full_response: boolean
:param decode_response_body: If True (and *want_full_response* is False), the response body is decoded and, if it is a JSON string, deserialized. Otherwise, the response body is uncompressed if transport compression is on, and returned raw.
:type decode_response_body: boolean
:param prepend_srv: If True, prepends the API server location to the URL
:type prepend_srv: boolean
:param max_retries: Maximum number of retries to perform for a request. A "failed" request is retried if any of the following is true:
- A response is received from the server, and the content length received does not match the "Content-Length" header.
- A response is received from the server, and the response has an HTTP status code in 5xx range.
- A response is received from the server, the "Content-Length" header is not set, and the response JSON cannot be parsed.
- No response is received from the server, and either *always_retry* is True or the request *method* is "GET".
:type max_retries: int
:param always_retry: If True, indicates that it is safe to retry a request on failure
- Note: It is not guaranteed that the request will *always* be retried on failure; rather, this is an indication to the function that it would be safe to do so.
:type always_retry: boolean
:returns: Response from API server in the format indicated by *want_full_response* and *decode_response_body*.
:raises: :exc:`exceptions.DXAPIError` or a subclass if the server returned a non-200 status code; :exc:`requests.exceptions.HTTPError` if an invalid response was received from the server; or :exc:`requests.exceptions.ConnectionError` if a connection cannot be established.
Wrapper around :meth:`requests.request()` that makes an HTTP
request, inserting authentication headers and (by default)
converting *data* to JSON.
.. note:: Bindings methods that make API calls make the underlying
HTTP request(s) using :func:`DXHTTPRequest`, and most of them
will pass any unrecognized keyword arguments you have supplied
through to :func:`DXHTTPRequest`.
'''
if session_handler is None:
session_handler = SESSION_HANDLERS[os.getpid()]
if headers is None:
headers = {}
global _UPGRADE_NOTIFY
url = APISERVER + resource if prepend_srv else resource
method = method.upper(
) # Convert method name to uppercase, to ease string comparisons later
if _DEBUG >= 3:
print(method,
url,
"=>\n" + json.dumps(data, indent=2),
file=sys.stderr)
elif _DEBUG == 2:
print(method, url, "=>", json.dumps(data), file=sys.stderr)
elif _DEBUG > 0:
from repr import Repr
print(method, url, "=>", Repr().repr(data), file=sys.stderr)
if auth is True:
auth = AUTH_HELPER
if 'verify' not in kwargs and 'DX_CA_CERT' in os.environ:
kwargs['verify'] = os.environ['DX_CA_CERT']
if os.environ['DX_CA_CERT'] == 'NOVERIFY':
kwargs['verify'] = False
from requests.packages import urllib3
urllib3.disable_warnings()
if jsonify_data:
data = json.dumps(data)
if 'Content-Type' not in headers and method == 'POST':
headers['Content-Type'] = 'application/json'
headers['DNAnexus-API'] = API_VERSION
headers['User-Agent'] = USER_AGENT
if use_compression == 'snappy':
if not snappy_available:
raise exceptions.DXError(
"Snappy compression requested, but the snappy module is unavailable"
)
headers['accept-encoding'] = 'snappy'
# If the input is a buffer, its data gets consumed by
# requests.request (moving the read position). Record the initial
# buffer position so that we can return to it if the request fails
# and needs to be retried.
rewind_input_buffer_offset = None
if hasattr(data, 'seek') and hasattr(data, 'tell'):
rewind_input_buffer_offset = data.tell()
try_index = 0
while True:
success, streaming_response_truncated = True, False
response = None
try:
_method, _url, _headers = _process_method_url_headers(
method, url, headers)
response = session_handler.request(_method,
_url,
headers=_headers,
data=data,
timeout=timeout,
auth=auth,
**kwargs)
if _UPGRADE_NOTIFY and response.headers.get(
'x-upgrade-info', '').startswith(
'A recommended update is available'
) and not os.environ.has_key('_ARGCOMPLETE'):
logger.info(response.headers['x-upgrade-info'])
try:
with file(_UPGRADE_NOTIFY, 'a'):
os.utime(_UPGRADE_NOTIFY, None)
except:
pass
_UPGRADE_NOTIFY = False
# If an HTTP code that is not in the 200 series is received and the content is JSON, parse it and throw the
# appropriate error. Otherwise, raise the usual exception.
if response.status_code // 100 != 2:
# response.headers key lookup is case-insensitive
if response.headers.get('content-type',
'').startswith('application/json'):
content = json.loads(response.content.decode('utf-8'))
error_class = getattr(exceptions, content["error"]["type"],
exceptions.DXAPIError)
raise error_class(content, response.status_code)
response.raise_for_status()
if want_full_response:
return response
else:
if 'content-length' in response.headers:
if int(response.headers['content-length']) != len(
response.content):
range_str = (
' (%s)' %
(headers['Range'], )) if 'Range' in headers else ''
raise exceptions.ContentLengthError(
"Received response with content-length header set to %s but content length is %d%s"
% (response.headers['content-length'],
len(response.content), range_str))
if use_compression and response.headers.get(
'content-encoding', '') == 'snappy':
# TODO: check if snappy raises any exceptions on truncated response content
content = snappy.uncompress(response.content)
else:
content = response.content
if decode_response_body:
content = content.decode('utf-8')
if response.headers.get('content-type',
'').startswith('application/json'):
try:
content = json.loads(content)
t = int(response.elapsed.total_seconds() * 1000)
if _DEBUG >= 3:
print(method,
url,
"<=",
response.status_code,
"(%dms)" % t,
"\n" + json.dumps(content, indent=2),
file=sys.stderr)
elif _DEBUG == 2:
print(method,
url,
"<=",
response.status_code,
"(%dms)" % t,
json.dumps(content),
file=sys.stderr)
elif _DEBUG > 0:
print(method,
url,
"<=",
response.status_code,
"(%dms)" % t,
Repr().repr(content),
file=sys.stderr)
return content
except ValueError:
# If a streaming API call (no content-length
# set) encounters an error it may just halt the
# response because it has no other way to
# indicate an error. Under these circumstances
# the client sees unparseable JSON, and we
# should be able to recover.
streaming_response_truncated = 'content-length' not in response.headers
raise HTTPError(
"Invalid JSON received from server")
return content
raise AssertionError(
'Should never reach this line: expected a result to have been returned by now'
)
except Exception as e:
success = False
exception_msg = _extract_msg_from_last_exception()
if isinstance(e, _expected_exceptions):
if response is not None and response.status_code == 503:
seconds_to_wait = _extract_retry_after_timeout(response)
logger.warn(
"%s %s: %s. Waiting %d seconds due to server unavailability...",
method, url, exception_msg, seconds_to_wait)
time.sleep(seconds_to_wait)
# Note, we escape the "except" block here without
# incrementing try_index because 503 responses with
# Retry-After should not count against the number of
# permitted retries.
continue
# Total number of allowed tries is the initial try + up to
# (max_retries) subsequent retries.
total_allowed_tries = max_retries + 1
ok_to_retry = False
# Because try_index is not incremented until we escape this
# iteration of the loop, try_index is equal to the number of
# tries that have failed so far, minus one. Test whether we
# have exhausted all retries.
if try_index + 1 < total_allowed_tries:
if response is None or isinstance(e, exceptions.ContentLengthError) or \
streaming_response_truncated:
ok_to_retry = always_retry or (
method == 'GET') or _is_retryable_exception(e)
else:
ok_to_retry = 500 <= response.status_code < 600
if ok_to_retry:
if rewind_input_buffer_offset is not None:
data.seek(rewind_input_buffer_offset)
delay = min(2**try_index, DEFAULT_TIMEOUT)
logger.warn(
"%s %s: %s. Waiting %d seconds before retry %d of %d...",
method, url, exception_msg, delay, try_index + 1,
max_retries)
time.sleep(delay)
try_index += 1
continue
# All retries have been exhausted OR the error is deemed not
# retryable. Print the latest error and propagate it back to the caller.
if not isinstance(e, exceptions.DXAPIError):
logger.error("%s %s: %s", method, url, exception_msg)
raise
finally:
if success and try_index > 0:
logger.info("%s %s: Recovered after %d retries", method, url,
try_index)
raise AssertionError(
'Should never reach this line: should have attempted a retry or reraised by now'
)
raise AssertionError(
'Should never reach this line: should never break out of loop')
from requests import request, Session
from requests.packages import urllib3
from requests.exceptions import RequestException
from .Exceptions import HTTPError, SiadError
# Disable pyopenssl. It breaks SSL connection pool when SSL connection is
# closed unexpetedly by the server. And we don't need SNI anyway.
try:
from requests.packages.urllib3.contrib import pyopenssl
pyopenssl.extract_from_urllib3()
except ImportError:
pass
# Disable SNI related Warning. The API does not rely on it
urllib3.disable_warnings(urllib3.exceptions.SNIMissingWarning)
urllib3.disable_warnings(urllib3.exceptions.SecurityWarning)
#: Default timeout for each request.
TIMEOUT = 5
class Client(object):
"""
"""
def __init__(self, endpoint="127.0.0.1:9980", sia_agent="Sia-Agent", force_discover=False, auth_key=""):
# Use Requests.Session to avoid any problem during the communication
self._session = Session()
self._endpoint = endpoint
#-*- coding:utf8 -*-
__author = "huia"
import requests
response = requests.get('https://www.12306.cn')
print(response.status_code)
import requests
from requests.packages import urllib3
urllib3.disable_warnings() #不显示未验证的警告
response = requests.get('https://www.12306.cn', verify=False) #verify是否验证证书
print(response.status_code)
#指定证书
import requests
response = requests.get('https://www.12306.cn',
cert=('/path/server.crt', '/path/key'))
print(response.status_code)
def getTweets():
time_per_100 = []
fetch_count = 0
avg_time_per_fetch = 0
tweet_count = 0
modified_at = None
# disable all HTTPS/TLS warnings
urllib3.disable_warnings()
# load api keys from api_keys.json
keys_file_path = os.path.join(project_root, 'Key', 'api_keys.json')
with open(keys_file_path) as api_keys:
keys = json.load(api_keys)
# obtain multiple instances of Twitter API to circumvent rate limit
authList = []
apiList = []
for i in range(len(keys)):
authList[i] = tweepy.OAuthHandler(keys[i]['consumer_key'],
keys[i]['consumer_secret'])
authList[i].set_access_token(keys[i]['access_token'],
keys[i]['access_token_secret'])
apiList[i] = tweepy.API(authList[i])
# db_path = os.path.join(os.path.dirname(__file__), os.pardir, 'Data/tweet_ids')
# connect to DB
db = connect('Shit_db')
# drop the database once to ensure whenever the thread is run, we create the db afresh
db.drop_database('Shit_db')
t0 = time.time()
no_of_tweets = 0
total_no_of_tweets = 0
# news channel IDs
# user_id = [34908698,362051343,361501426,7905122,180306960,30857481,28370738,110458336,2883841,28172926,30261067,20562637,113050195,28140646,621523,35773039,15164565,15861355,44316192,44078873,15861220,1642135962,28137012,38400130,32355144,122097236,19230601,713993413,7302282,16877611,2557521,26257166,15110357,4898091,34713362,18949452,32359921,16334857,59736898,214007688,129834917,15108702,39817941,375721095,2424208849,506504366,242689065,116559622,23484039,18424289,64643056,115754870,134758540,6509832,267158021,29958928,15954704,19897138,37034483,36327407,20751449,3123883238,240649814,31632905,177829660,256495314,39743812,245687754,38647512,355989081,98362607,17710740,39240673,17469289,16973333,87818409,18071358,9763482,87416722,4970411,7587032,788524,14173315,612473,28785486,2467791,15012486,5988062,1367531,759251,428333,6017542,3108351,51241574,1652541,14293310,807095,742143,5402612]
# non news channel IDs
user_id = [
79708561, 281766200, 785493949, 250205792, 180463340, 3060210854,
2305049443, 273181052, 2463499796, 71876190, 26642006, 92367751,
259379883, 399428964, 26565946, 24494557, 166739404, 52551600,
25365536, 15485441, 15846407, 14234323, 125481462, 27042513, 133880286,
243284052, 44588485, 51376979, 27260086, 17919972, 18625669, 16409683,
21447363, 58135085, 23375688, 92724677, 30973, 50374439, 48410093,
57928790, 87170183, 102957248, 108391251, 120998613, 115622213,
113419517, 6463042, 94775494, 131975194, 97865628, 79915337, 332188446,
41067945, 197150180, 78022296, 31348594, 902534288, 108253263,
63390627, 145125358, 78242874, 468479147, 36057824, 34464376,
111871312, 152251488, 121677709, 38403110, 21787625, 494747331,
94163409, 44849431, 18872373, 105710210, 148248527, 38479920,
508932270, 183230911, 186388502, 101311381, 70652594, 2719753171,
23976386, 23002923, 33868638, 16548023, 40453512, 18681139, 279449435,
144755081, 132385468, 54829997, 266714730, 108252113, 3138637447,
1111706414, 61755650, 14120922, 216447259, 129786468
]
print "No of users={0}".format(len(user_id))
last_id = [None for i in range(len(user_id))]
number = 0
rate_limit = 180
no_of_requests = 0
tweet_list = []
k = 0 # current_api_index
api_wait_end_time = time.time(
) # stores timestamp till when the first API might have to wait
while (total_no_of_tweets < 3200 * len(user_id)):
try:
status_obj = apiList[k].user_timeline(user_id=user_id[number],
count=200,
max_id=last_id[number])
# print "fetched {0} tweets".format(len(status_obj))
no_of_requests += 1
for status in status_obj:
tweet = Tweet()
tweet.tweet_id = status.id_str
tweet.text = status.text
tweet.created_at = status.created_at
tweet.in_reply_to_status_id = status.in_reply_to_status_id_str
tweet.user_id = status.user.id_str
tweet.user_name = status.user.name
tweet.user_followers = status.user.followers_count
tweet.user_location = status.user.location
tweet.favourites_count = status.user.favourites_count
if status.coordinates is not None:
tweet.coordinates = status.coordinates['coordinates']
tweet.language = status.lang
# tweet.place_coordinates = status['']
tweet.retweet_count = status.retweet_count
tweet.retweeted = status.retweeted
# tweet.inserted_at
tweet.is_news = True
# tweet.save()
tweet_list.append(tweet)
no_of_tweets = no_of_tweets + 1
total_no_of_tweets = total_no_of_tweets + 1
last_id[number] = tweet.tweet_id
#print(tweet.user_name)
# print "last id={0}".format(last_id[number])
# print "total no of tweets {0}".format(no_of_tweets)
if no_of_requests % 100 == 0:
print "{0} tweets fetched".format(total_no_of_tweets)
if (no_of_tweets >= 3200):
print "Saving Tweets to DB"
# save tweets to db
Tweet.objects.insert(tweet_list)
tweet_list = []
number += 1
# if we have fetched tweets for every user, just return
if number > len(user_id):
return
number = number % len(user_id)
print "moved to {0} user".format(number)
no_of_tweets = 0
except tweepy.RateLimitError:
print "Saving Tweets to DB"
# save tweets to db
Tweet.objects.insert(tweet_list)
tweet_list = []
if k == len(apiList) - 1:
if api_wait_end_time < time.time():
# we dont need to wait, so pass
pass
else:
sleep_time = api_wait_end_time - time.time()
print "create_db: sleeping for {0} seconds".format(
sleep_time)
time.sleep(sleep_time)
k = (k + 1) % len(apiList)
if k == 0:
# update api_wait_end_time
api_wait_end_time = time.time() + 15 * 60
# print("Going to Sleep")
# print no_of_requests
# t0 = time.time() - t0
# if t0 > 16*60:
# print "sleeping for {0} sec".format(15*60)
# time.sleep(15*60)
# else:
# print "sleeping for {0} sec".format(16*60 - t0)
# time.sleep(16*60 - t0)
# t0 = time.time()
except Exception as e:
print("exception came")
print(str(e))
time.sleep(15 * 60)
t0 = time.time() - t0
t0 = t0 / 60
print(t0)
return
class BaseClient(object):
"""
Basic API client
- Supports Authorization with tokens
- Caches tokens
"""
disable_warnings(InsecurePlatformWarning)
disable_warnings(InsecureRequestWarning)
disable_warnings(SNIMissingWarning)
_logger = Logger('api')
def __init__(self,
ip,
port,
credentials=None,
verify=False,
version='*',
raw_response=False,
cache_store=None):
"""
Initializes the object with credentials and connection information
:param ip: IP to which to connect
:type ip: str
:param port: Port on which to connect
:type port: int
:param credentials: Credentials to connect
:type credentials: tuple
:param verify: Additional verification
:type verify: bool
:param version: API version
:type version: object
:param raw_response: Retrieve the raw response value
:type raw_response: bool
:param cache_store: Store in which to keep the generated token for the client instance
:type cache_store: any
:return: None
:rtype: NoneType
"""
if credentials is not None and len(credentials) != 2:
raise RuntimeError(
'Credentials should be None (no authentication) or a tuple containing client_id and client_secret (authenticated)'
)
self.ip = ip
self.port = port
self.client_id = credentials[0] if credentials is not None else None
self.client_secret = credentials[1] if credentials is not None else None
self._url = 'https://{0}:{1}/api'.format(ip, port)
self._key = hashlib.sha256('{0}{1}{2}{3}'.format(
self.ip, self.port, self.client_id,
self.client_secret)).hexdigest()
self._token = None
self._verify = verify
self._version = version
self._raw_response = raw_response
self._volatile_client = cache_store
def _connect(self):
"""
Authenticates to the api
"""
headers = {
'Accept':
'application/json',
'Authorization':
'Basic {0}'.format(
base64.b64encode('{0}:{1}'.format(self.client_id,
self.client_secret)).strip())
}
raw_response = requests.post(url='{0}/oauth2/token/'.format(self._url),
data={'grant_type': 'client_credentials'},
headers=headers,
verify=self._verify)
try:
response = self._process(response=raw_response, overrule_raw=True)
except RuntimeError:
if self._raw_response is True:
return raw_response
raise
if len(response.keys()) in [1, 2] and 'error' in response:
error = RuntimeError(response['error'])
error.status_code = raw_response.status_code
raise error
self._token = response['access_token']
def _build_headers(self):
"""
Builds the request headers
:return: The request headers
:rtype: dict
"""
headers = {
'Accept': 'application/json; version={0}'.format(self._version),
'Content-Type': 'application/json'
}
if self._token is not None:
headers['Authorization'] = 'Bearer {0}'.format(self._token)
return headers
@classmethod
def _build_url_params(cls, params=None):
"""
Build the URL params
:param params: URL parameters
:type params: str
:return: The url params
:rtype: string
"""
url_params = ''
if params:
url_params = '?{0}'.format(urllib.urlencode(params))
return url_params
def _cache_token(self):
"""
Caches the JWT
:return: None
:rtype: NoneType
"""
if self._volatile_client is not None:
self._volatile_client.set(self._key, self._token, 300)
def _prepare(self, **kwargs):
"""
Prepares the call:
* Authentication, if required
* Preparing headers, returning them
"""
if self.client_id is not None and self._token is None:
self._connect()
headers = self._build_headers()
params = self._build_url_params(kwargs.get('params'))
url = '{0}{{0}}{1}'.format(self._url, params)
self._cache_token(
) # Volatile cache might have expired or the key is gone
return headers, url
def _process(self, response, overrule_raw=False):
"""
Processes a call result
"""
if self._raw_response is True and overrule_raw is False:
return response
status_code = response.status_code
parsed_output = None
try:
parsed_output = response.json()
except:
pass
if 200 <= status_code < 300:
return parsed_output
else:
message = None
if parsed_output is not None:
if 'error_description' in parsed_output:
message = parsed_output['error_description']
if 'error' in parsed_output:
if message is None:
message = parsed_output['error']
else:
message += ' ({0})'.format(parsed_output['error'])
else:
messages = {
401: 'No access to the requested API',
403: 'No access to the requested API',
404: 'The requested API could not be found',
405: 'Requested method not allowed',
406: 'The request was unacceptable',
426: 'Upgrade is needed',
429: 'Rate limit was hit',
500: 'Internal server error'
}
if status_code in messages:
message = messages[status_code]
if message is None:
message = 'Unknown error'
if status_code in [401, 403]:
raise HttpForbiddenException(message, '')
elif status_code == 404:
raise HttpNotFoundException(message, '')
else:
raise HttpException(status_code, message)
def _call(self, api, params, fct, timeout=None, **kwargs):
if not api.endswith('/'):
api = '{0}/'.format(api)
if not api.startswith('/'):
api = '/{0}'.format(api)
if self._volatile_client is not None:
self._token = self._volatile_client.get(self._key)
first_connect = self._token is None
headers, url = self._prepare(params=params)
try:
return self._process(
fct(url=url.format(api),
headers=headers,
verify=self._verify,
timeout=timeout,
**kwargs))
except HttpForbiddenException:
if self._volatile_client is not None:
self._volatile_client.delete(self._key)
if first_connect is True: # First connect, so no token was present yet, so no need to try twice without token
raise
self._token = None
headers, url = self._prepare(params=params)
return self._process(
fct(url=url.format(api),
headers=headers,
verify=self._verify,
**kwargs))
except Exception:
if self._volatile_client is not None:
self._volatile_client.delete(self._key)
raise
@classmethod
def get_instance(cls, connection_info, cache_store=None, version=6):
"""
Retrieve an OVSClient instance to the connection information passed
:param connection_info: Connection information, includes: 'host', 'port', 'client_id', 'client_secret'
:type connection_info: dict
:param cache_store: Store in which to keep the generated token for the client
:type cache_store: object
:param version: Version for the API
:type version: int
:return: An instance of the OVSClient class
:rtype: ovs_extensions.api.client.OVSClient
"""
ExtensionsToolbox.verify_required_params(
actual_params=connection_info,
required_params={
'host': (str, ExtensionsToolbox.regex_ip),
'port': (int, {
'min': 1,
'max': 65535
}),
'client_id': (str, None),
'client_secret': (str, None),
'local': (bool, None, False)
})
return cls(ip=connection_info['host'],
port=connection_info['port'],
credentials=(connection_info['client_id'],
connection_info['client_secret']),
version=version,
cache_store=cache_store)
def get(self, api, params=None):
"""
Executes a GET call
:param api: Specification to fill out in the URL, eg: /vpools/<vpool_guid>/shrink_vpool
:param params: Additional query parameters as comma separated list, eg: {'contents':'dynamic1,dynamic2,-dynamic3,_relations,-relation1'}
:type params: dict
"""
return self._call(api=api, params=params, fct=requests.get)
def post(self, api, data=None, params=None):
"""
Executes a POST call
:param api: Specification to fill out in the URL, eg: /vpools/<vpool_guid>/shrink_vpool
:param data: Data to post
:param params: Additional query parameters, eg: _dynamics
"""
return self._call(api=api, params=params, fct=requests.post, data=data)
def put(self, api, data=None, params=None):
"""
Executes a PUT call
:param api: Specification to fill out in the URL, eg: /vpools/<vpool_guid>/shrink_vpool
:param data: Data to put
:param params: Additional query parameters, eg: _dynamics
"""
return self._call(api=api, params=params, fct=requests.put, data=data)
def patch(self, api, data=None, params=None):
"""
Executes a PATCH call
:param api: Specification to fill out in the URL, eg: /vpools/<vpool_guid>/shrink_vpool
:param data: Data to patch
:param params: Additional query parameters, eg: _dynamics
"""
return self._call(api=api,
params=params,
fct=requests.patch,
data=data)
def delete(self, api, params=None):
"""
Executes a DELETE call
:param api: Specification to fill out in the URL, eg: /vpools/<vpool_guid>/
:param params: Additional query parameters, eg: _dynamics
"""
return self._call(api=api, params=params, fct=requests.delete)
def main():
# We silence the insecure requests warnings we get for using
# self-signed certificates.
disable_warnings(InsecureRequestWarning)
cli(obj=RemoteAppRestContext())
def webapi(
url,
hostname_override=None,
principal=None,
authtype='kerberos',
username=None,
password=None,
verifyssl=False,
error_action='Stop'):
"""Sends a request to the provided url and authentication configuration.
If successfully, a WebAPI object will be constructed from the response
and returned.
:param url: The URL of the PI Web API server.
:param authtype: Optional - Options are Basic and Kerberos.
Default authtype is kerberos.
:param username: Optional username - Only used for basic auth.
:param password: Optional password - Only used for basic auth.
:param verifyssl: Optional SSL verification. If set to False, then
InsecureRequestWarning will be disabled.
:return: :class:`WebAPI <WebAPI>` object
:rtype: osisoftpy.WebAPI
"""
try:
s = requests.session()
s.verify = verifyssl
if not s.verify:
disable_warnings(InsecureRequestWarning)
if authtype == 'kerberos':
s.auth = HTTPKerberosAuth(
mutual_authentication=requests_kerberos.OPTIONAL,
sanitize_mutual_error_response=False,
hostname_override=hostname_override,
force_preemptive=True,
principal=principal)
else:
s.auth = requests.auth.HTTPBasicAuth(username, password)
r = APIResponse(s.get(url), s)
if r.response.status_code == 401:
msg = 'Authorization denied - incorrect username or password.'
if error_action.lower() == 'stop':
raise Unauthorized(msg)
else:
print(msg + ', Continuing')
if r.response.status_code != 200:
msg = 'Wrong server response: %s %s' % (r.response.status_code, r.response.reason)
if error_action.lower() == 'stop':
raise HTTPError(msg)
else:
print(msg + ', Continuing')
json = r.response.json()
if 'Errors' in json and json.get('Errors').__len__() > 0:
msg = 'PI Web API returned an error: {}'
raise PIWebAPIError(msg.format(json.get('Errors')))
webapi = create(Factory(WebAPI), json, r.session)
webapi.dataservers = _get_servers(webapi, s, json, DataServer, 'DataServers')
webapi.assetservers = _get_servers(webapi, s, json, AssetServer, 'AssetServers')
return webapi
except:
raise
from __future__ import absolute_import from requests.packages import urllib3 urllib3.disable_warnings() # noqa from dataverse.connection import Connection # noqa from dataverse.dataverse import Dataverse # noqa from dataverse.dataset import Dataset # noqa from dataverse.file import DataverseFile # noqa
def main(*args):
import argparse
import getpass
import sys
parser = argparse.ArgumentParser(description='request a SAML token')
parser.add_argument('-u', '--user', help="account user name")
parser.add_argument('-p', '--password', help="account password")
parser.add_argument('-r',
'--raw',
action='store_true',
help="don't pack and wrap the token")
parser.add_argument('--insecure',
action='store_true',
help="disable SSL/TLS security checks")
parser.add_argument('--cert-only',
action='store_true',
help="output embedded certificates in PEM form")
# compatibility argument -- we don't print out anything
parser.add_argument('-s',
'--silent',
action='store_true',
help=argparse.SUPPRESS,
default=argparse.SUPPRESS)
options = parser.parse_args()
def my_input(prompt):
sys.stderr.write(prompt)
return input()
username = options.user or my_input('User: '******'Password: '******'SSL request failed; you probably need to install the '
'appropriate certificate authority, or use the correct host '
'name')
print(msg, e, file=sys.stderr)
return 1
if not options.cert_only:
sys.stdout.write(token)
else:
from lxml import etree
import ssl
for el in etree.fromstring(token).findall('.//{*}X509Certificate'):
data = base64.standard_b64decode(el.text)
sys.stdout.write(ssl.DER_cert_to_PEM_cert(data))
return 0
# wrong demo
requests.get('http://httpbin.org/cookies/set/number/123456789')
r = requests.get('http://httpbin.org/cookies')
print(r.text) # {"cookies": {}} no cookies received
# right demo
s = requests.Session()
s.get('http://httpbin.org/cookies/set/number/123456789')
r = s.get('http://httpbin.org/cookies')
print(r.text)
"""SSL certificate authentication"""
# fail to validate certificate
response = requests.get('https://www.12306.cn', verify=False)
print(response.status_code)
# ignore waring
urllib3.disable_warnings() # logging.captureWarnings(True)
response = requests.get('https://www.12306.cn', verify=False)
print(response.status_code)
# use certificate
response = requests.get('https://www.12306.cn',
cert=('/path/server.crt', '/path/key'))
print(response.status_code)
"""Proxy Setting"""
# demo
proxies = {'http': 'http...', 'https': 'https...'}
# use HTTP Basic Auth
proxies = {'http': 'http://*****:*****@...', 'https': '...'}
# use socks
# pip3 install requests[socks]
proxies = {'http': 'socks5://user:password@...', 'https': 'socks5://...'}
import requests
import queue
import re
import datetime
from requests.packages import urllib3
urllib3.disable_warnings()
#健康优选版
url = 'https://www.baodan360.com/insurance/detail/id/133339/'
Sex = ['男', '女']
Extra = ['含附加被保人豁免', '不含附加被保人豁免']
Insurance_Period = ['至80周岁', '保30年', '至60周岁', '至70周岁']
Pay_Period = ['1', '5', '10', '20', '30']
Insurance_Amount = [
x * 10 for x in range(1, 21)
if x != 4 and x != 7 and x != 9 and x != 14 and x != 19
]
First_City = [110100, 310100, 440100, 440300, 320500, 330200, 330100, 320100]
Second_City = [
110100, 310100, 120100, 500100, 230100, 220100, 210100, 150100, 130100,
650100, 620100, 630100, 610100, 640100, 410100, 370100, 140100, 340100,
420100, 430100, 320100, 510100, 520100, 530100, 450100, 330100, 360100,
440100, 350100, 710000, 810000, 820000, 460100, 150200, 210200, 220200,
230600, 410300, 420500, 430700, 450200, 510700, 520300, 340200, 130200,
130600, 130900, 350200, 350500, 350600, 330300, 330400, 330600, 330700,
331000, 440400, 440600, 440700, 440800, 440900, 441300, 441900, 442000,
371600, 371700, 370200, 370300, 370500, 370600, 370700, 370800, 370900,
371000, 371300, 371400, 371500, 320200, 320300, 320400, 320600, 320800,
320900, 321000, 321100, 321200
]
RESULTS = []
dictionary_standard = {}
def scrapeTweets():
time_per_100 = []
fetch_count = 0
avg_time_per_fetch = 0
tweet_count = 0
target = 1000000
modified_at = None
# disable all HTTPS/TLS warnings
urllib3.disable_warnings()
# load api keys from api_keys.json
keys_file_path = os.path.join(os.path.dirname(__file__), os.pardir, 'api_keys.json')
with open(keys_file_path) as api_keys:
keys = json.load(api_keys)
# provide auth params & obtain an instance of API
auth = tweepy.OAuthHandler(keys['consumer_key'], keys['consumer_secret'])
auth.set_access_token(keys['access_token'], keys['access_token_secret'])
api = tweepy.API(auth)
db_path = os.path.join(os.path.dirname(__file__), os.pardir, 'Data/tweet_ids')
# connect to DB
db = connect('Tweets')
# drop the database once to ensure whenever the thread is run, we create the db afresh
db.drop_database('Tweets')
tweet_id_list = []
with open(db_path) as file_db:
t0 = time.time()
for line in file_db:
status_obj = None
tweet_id = line.split("\t")[0]
tweet_id_list.append(tweet_id)
if(len(tweet_id_list) == 100):
try:
status_obj = api.statuses_lookup(tweet_id_list, [False], [False], [True])
for status in status_obj:
tweet = Tweet()
tweet.tweet_id = status.id_str
tweet.text = status.text
tweet.created_at = status.created_at
tweet.in_reply_to_status_id = status.in_reply_to_status_id_str
tweet.user_id = status.user.id_str
tweet.user_name = status.user.name
tweet.user_followers = status.user.followers_count
tweet.user_location = status.user.location
tweet.favourites_count = status.user.favourites_count
if status.coordinates is not None:
tweet.coordinates = status.coordinates['coordinates']
tweet.language = status.lang
# tweet.place_coordinates = status['']
tweet.retweet_count = status.retweet_count
tweet.retweeted = status.retweeted
# tweet.inserted_at
tweet.is_news = None
tweet.save()
t1 = time.time()
time_per_100.append(t1-t0)
fetch_count = fetch_count + 1
avg_time_per_fetch = sum(time_per_100)/len(time_per_100)
tweet_count += len(status_obj)
modified_at = datetime.datetime.now().strftime('%H:%M:%S %d-%m-%Y')
print("Scraped {0} tweets, Total ={1} tweets".format(
len(status_obj), tweet_count))
# save all the stats to REDIS
r.set('tweet_count', tweet_count)
r.set('avg_time_per_fetch', avg_time_per_fetch)
r.set('fetch_count', fetch_count)
r.set('modified_at', modified_at)
# r.set('target', target)
except tweepy.RateLimitError:
print("Going to Sleep")
time.sleep(15 * 60)
except Exception as e:
print(str(e))
time.sleep(15 * 60)
finally:
from rucio import version
try:
# Python 2
from urlparse import urlparse
from ConfigParser import NoOptionError, NoSectionError
except ImportError:
# Python 3
from urllib.parse import urlparse
from configparser import NoOptionError, NoSectionError
from dogpile.cache import make_region
from requests import Session
from requests.status_codes import codes, _codes
from requests.exceptions import ConnectionError, RequestException
from requests.packages.urllib3 import disable_warnings # pylint: disable=import-error
disable_warnings()
# Extra modules: Only imported if available
EXTRA_MODULES = {'requests_kerberos': False}
for extra_module in EXTRA_MODULES:
try:
imp.find_module(extra_module)
EXTRA_MODULES[extra_module] = True
except ImportError:
EXTRA_MODULES[extra_module] = False
if EXTRA_MODULES['requests_kerberos']:
from requests_kerberos import HTTPKerberosAuth # pylint: disable=import-error
LOG = getLogger(__name__)
class EtcdConfiguration(object):
"""
Configuration class using Etcd.
Uses a special key format to specify the path within etcd, and specify a path inside the json data
object that might be stored inside the etcd key.
key = <etcd path>[|<json path>]
etcd path = slash-delimited path
json path = dot-delimited path
Examples:
> EtcdConfiguration.set('/foo', 1)
> print EtcdConfiguration.get('/foo')
< 1
> EtcdConfiguration.set('/foo', {'bar': 1})
> print EtcdConfiguration.get('/foo')
< {u'bar': 1}
> print EtcdConfiguration.get('/foo|bar')
< 1
> EtcdConfiguration.set('/bar|a.b', 'test')
> print EtcdConfiguration.get('/bar')
< {u'a': {u'b': u'test'}}
"""
_unittest_data = {}
base_config = {
'cluster_id': None,
'external_etcd': None,
'plugins/installed': {
'backends': [],
'generic': []
},
'paths': {
'cfgdir': '/opt/OpenvStorage/config',
'basedir': '/opt/OpenvStorage',
'ovsdb': '/opt/OpenvStorage/db'
},
'support': {
'enablesupport': False,
'enabled': True,
'interval': 60
},
'storagedriver': {
'mds_safety': 2,
'mds_tlogs': 100,
'mds_maxload': 75
},
'webapps': {
'html_endpoint': '/',
'oauth2': {
'mode': 'local'
}
}
}
disable_warnings(InsecurePlatformWarning)
disable_warnings(InsecureRequestWarning)
disable_warnings(SNIMissingWarning)
def __init__(self):
"""
Dummy init method
"""
_ = self
@staticmethod
def get(key, raw=False):
"""
Get value from etcd
:param key: Key to get
:param raw: Raw data if True else json format
:return: Value for key
"""
key_entries = key.split('|')
data = EtcdConfiguration._get(key_entries[0], raw)
if len(key_entries) == 1:
return data
temp_data = data
for entry in key_entries[1].split('.'):
temp_data = temp_data[entry]
return temp_data
@staticmethod
def set(key, value, raw=False):
"""
Set value in etcd
:param key: Key to store
:param value: Value to store
:param raw: Raw data if True else json format
:return: None
"""
key_entries = key.split('|')
if len(key_entries) == 1:
EtcdConfiguration._set(key_entries[0], value, raw)
return
try:
data = EtcdConfiguration._get(key_entries[0], raw)
except etcd.EtcdKeyNotFound:
data = {}
temp_config = data
entries = key_entries[1].split('.')
for entry in entries[:-1]:
if entry in temp_config:
temp_config = temp_config[entry]
else:
temp_config[entry] = {}
temp_config = temp_config[entry]
temp_config[entries[-1]] = value
EtcdConfiguration._set(key_entries[0], data, raw)
@staticmethod
def delete(key, remove_root=False, raw=False):
"""
Delete key - value from etcd
:param key: Key to delete
:param remove_root: Remove root
:param raw: Raw data if True else json format
:return: None
"""
key_entries = key.split('|')
if len(key_entries) == 1:
EtcdConfiguration._delete(key_entries[0], recursive=True)
return
data = EtcdConfiguration._get(key_entries[0], raw)
temp_config = data
entries = key_entries[1].split('.')
if len(entries) > 1:
for entry in entries[:-1]:
if entry in temp_config:
temp_config = temp_config[entry]
else:
temp_config[entry] = {}
temp_config = temp_config[entry]
del temp_config[entries[-1]]
if len(entries) == 1 and remove_root is True:
del data[entries[0]]
EtcdConfiguration._set(key_entries[0], data, raw)
@staticmethod
def exists(key, raw=False):
"""
Check if key exists in etcd
:param key: Key to check
:param raw: Process raw data
:return: True if exists
"""
try:
EtcdConfiguration.get(key, raw)
return True
except (KeyError, etcd.EtcdKeyNotFound):
return False
@staticmethod
def dir_exists(key):
"""
Check if directory exists in etcd
:param key: Directory to check
:return: True if exists
"""
return EtcdConfiguration._dir_exists(key)
@staticmethod
def list(key):
"""
List all keys in tree
:param key: Key to list
:return: Generator object
"""
return EtcdConfiguration._list(key)
@staticmethod
def initialize_host(host_id, port_info=None):
"""
Initialize keys when setting up a host
:param host_id: ID of the host
:type host_id: str
:param port_info: Information about ports to be used
:type port_info: dict
:return: None
"""
if EtcdConfiguration.exists(
'/ovs/framework/hosts/{0}/setupcompleted'.format(host_id)):
return
if port_info is None:
port_info = {}
mds_port_range = port_info.get('mds', [26300, 26399])
arakoon_start_port = port_info.get('arakoon', 26400)
storagedriver_port_range = port_info.get('storagedriver',
[26200, 26299])
host_config = {
'storagedriver': {
'rsp': '/var/rsp',
'vmware_mode': 'ganesha'
},
'ports': {
'storagedriver': [storagedriver_port_range],
'mds': [mds_port_range],
'arakoon': [arakoon_start_port]
},
'setupcompleted': False,
'versions': {
'ovs': 4
},
'type': 'UNCONFIGURED'
}
for key, value in host_config.iteritems():
EtcdConfiguration._set('/ovs/framework/hosts/{0}/{1}'.format(
host_id, key),
value,
raw=False)
@staticmethod
def initialize(external_etcd=None, logging_target=None):
"""
Initialize general keys for all hosts in cluster
:param external_etcd: ETCD runs on another host outside the cluster
:param logging_target: Configures (overwrites) logging configuration
"""
cluster_id = ''.join(
random.choice(string.ascii_letters + string.digits)
for _ in range(16))
if EtcdConfiguration.exists('/ovs/framework/cluster_id'):
return
messagequeue_cfg = {
'endpoints': [],
'metadata': {
'internal': True
},
'protocol': 'amqp',
'user': '******',
'password': '******',
'queues': {
'storagedriver': 'volumerouter'
}
}
base_cfg = copy.deepcopy(EtcdConfiguration.base_config)
base_cfg.update({
'cluster_id': cluster_id,
'external_etcd': external_etcd,
'arakoon_clusters': {},
'stores': {
'persistent': 'pyrakoon',
'volatile': 'memcache'
},
'messagequeue': {
'protocol': 'amqp',
'queues': {
'storagedriver': 'volumerouter'
}
},
'logging': {
'type': 'console'
}
})
if logging_target is not None:
base_cfg['logging'] = logging_target
if EtcdConfiguration.exists('/ovs/framework/memcache') is False:
base_cfg['memcache'] = {
'endpoints': [],
'metadata': {
'internal': True
}
}
if EtcdConfiguration.exists('/ovs/framework/messagequeue') is False:
base_cfg['messagequeue'] = messagequeue_cfg
else:
messagequeue_info = EtcdConfiguration.get(
'/ovs/framework/messagequeue')
for key, value in messagequeue_cfg.iteritems():
if key not in messagequeue_info:
base_cfg['messagequeue'][key] = value
for key, value in base_cfg.iteritems():
EtcdConfiguration._set('/ovs/framework/{0}'.format(key),
value,
raw=False)
@staticmethod
@log_slow_calls
def _dir_exists(key):
key = EtcdConfiguration._coalesce_dashes(key=key)
# Unittests
if hasattr(unittest, 'running_tests') and getattr(
unittest, 'running_tests') is True:
stripped_key = key.strip('/')
current_dict = EtcdConfiguration._unittest_data
for part in stripped_key.split('/'):
if part not in current_dict or not isinstance(
current_dict[part], dict):
return False
current_dict = current_dict[part]
return True
# Real implementation
try:
client = EtcdConfiguration._get_client()
return client.get(key).dir
except (KeyError, etcd.EtcdKeyNotFound):
return False
@staticmethod
@log_slow_calls
def _list(key):
key = EtcdConfiguration._coalesce_dashes(key=key)
# Unittests
if hasattr(unittest, 'running_tests') and getattr(
unittest, 'running_tests') is True:
data = EtcdConfiguration._unittest_data
ends_with_dash = key.endswith('/')
starts_with_dash = key.startswith('/')
stripped_key = key.strip('/')
for part in stripped_key.split('/'):
if part not in data:
raise etcd.EtcdKeyNotFound(
'Key not found: {0}'.format(key))
data = data[part]
if data:
for sub_key in data:
if ends_with_dash is True:
yield '/{0}/{1}'.format(stripped_key, sub_key)
else:
yield sub_key if starts_with_dash is True else '/{0}'.format(
sub_key)
elif starts_with_dash is False or ends_with_dash is True:
yield '/{0}'.format(stripped_key)
return
# Real implementation
client = EtcdConfiguration._get_client()
for child in client.get(key).children:
if child.key is not None and child.key != key:
yield child.key.replace('{0}/'.format(key), '')
@staticmethod
@log_slow_calls
def _delete(key, recursive):
key = EtcdConfiguration._coalesce_dashes(key=key)
# Unittests
if hasattr(unittest, 'running_tests') and getattr(
unittest, 'running_tests') is True:
stripped_key = key.strip('/')
data = EtcdConfiguration._unittest_data
for part in stripped_key.split('/')[:-1]:
if part not in data:
raise etcd.EtcdKeyNotFound(
'Key not found : {0}'.format(key))
data = data[part]
key_to_remove = stripped_key.split('/')[-1]
if key_to_remove in data:
del data[key_to_remove]
return
# Real implementation
client = EtcdConfiguration._get_client()
client.delete(key, recursive=recursive)
@staticmethod
@log_slow_calls
def _get(key, raw):
key = EtcdConfiguration._coalesce_dashes(key=key)
# Unittests
if hasattr(unittest, 'running_tests') and getattr(
unittest, 'running_tests') is True:
if key in ['', '/']:
return
stripped_key = key.strip('/')
data = EtcdConfiguration._unittest_data
for part in stripped_key.split('/')[:-1]:
if part not in data:
raise etcd.EtcdKeyNotFound(
'Key not found : {0}'.format(key))
data = data[part]
last_part = stripped_key.split('/')[-1]
if last_part not in data:
raise etcd.EtcdKeyNotFound('Key not found : {0}'.format(key))
data = data[last_part]
if isinstance(data, dict):
data = None
else:
# Real implementation
client = EtcdConfiguration._get_client()
data = client.read(key).value
if raw is True:
return data
return json.loads(data)
@staticmethod
@log_slow_calls
def _set(key, value, raw):
key = EtcdConfiguration._coalesce_dashes(key=key)
data = value
if raw is False:
data = json.dumps(value)
# Unittests
if hasattr(unittest, 'running_tests') and getattr(
unittest, 'running_tests') is True:
stripped_key = key.strip('/')
ut_data = EtcdConfiguration._unittest_data
for part in stripped_key.split('/')[:-1]:
if part not in ut_data:
ut_data[part] = {}
ut_data = ut_data[part]
ut_data[stripped_key.split('/')[-1]] = data
return
# Real implementation
client = EtcdConfiguration._get_client()
client.write(key, data)
try:
def _escape(*args, **kwargs):
_ = args, kwargs
raise RuntimeError()
from ovs.extensions.storage.persistentfactory import PersistentFactory
client = PersistentFactory.get_client()
signal.signal(signal.SIGALRM, _escape)
signal.alarm(
0.5
) # Wait only 0.5 seconds. This is a backup and should not slow down the system
client.set(key, value)
signal.alarm(0)
except:
pass
@staticmethod
def _get_client():
return etcd.Client(port=2379, use_proxies=True)
@staticmethod
def _coalesce_dashes(key):
"""
Remove multiple dashes, eg: //ovs//framework/ becomes /ovs/framework/
:param key: Key to convert
:type key: str
:return: Key without multiple dashes after one another
:rtype: str
"""
return ''.join(k if k == '/' else ''.join(group)
for k, group in groupby(key))
from requests_kerberos import HTTPKerberosAuth
except Exception:
# requests_kerberos library require pykerberos.
# pykerberos require krb5-devel, which isn't python lib.
# Kerberos users will need to manually install it.
HTTPKerberosAuth = None
DEFAULT_PORT = 80
SECURED_PORT = 443
SECURED_PROTOCOL = 'https'
DEFAULT_PROTOCOL = 'http'
DEFAULT_API_VERSION = 'v3.1'
BASIC_AUTH_PREFIX = 'Basic'
CLOUDIFY_TENANT_HEADER = 'Tenant'
urllib3.disable_warnings(urllib3.exceptions.InsecurePlatformWarning)
class HTTPClient(object):
def __init__(self,
host,
port=DEFAULT_PORT,
protocol=DEFAULT_PROTOCOL,
api_version=DEFAULT_API_VERSION,
headers=None,
query_params=None,
cert=None,
trust_all=False,
username=None,
password=None,
token=None,
#
# This script delete policies spread over a number rics
# Intended for parallel processing
# Returns a string with result, either "0" for ok, or "1<fault description>"
import os
import json
import sys
import requests
import traceback
# disable warning about unverified https requests
from requests.packages import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
#arg responsecode baseurl num_rics uuid startid count pids pid_id proxy
try:
if len(sys.argv) != 10:
print("1Expected 9 args, got "+str(len(sys.argv)-1)+ ". Args: responsecode baseurl num_rics uuid startid count pids pid_id proxy")
sys.exit()
responsecode=int(sys.argv[1])
baseurl=str(sys.argv[2])
num_rics=int(sys.argv[3])
uuid=str(sys.argv[4])
start=int(sys.argv[5])
count=int(sys.argv[6])
pids=int(sys.argv[7])
def __init__(self, config):
"""
Initialization is responsible for fetching service instance objects for each vCenter instance
pyvmomi has some security checks enabled by default from python 2.7.9 onward to connect to vCenter.
"""
# Holds all the VMs' instanceuuid that are discovered for each of the vCenter. Going ahead it would hold all the
# other managed objects of vCenter that would be monitored.
self.mors = {}
self.params = config
global metrics
global counters
metrics = util.parse_metrics()
counters = util.parse_counters()
self.needed_metrics = {}
self.configured_metrics = {}
self.refresh_rates = {}
for k, v in metrics.items():
self.configured_metrics.update({util.get_counter(k): v})
if sys.version_info > (2, 7, 9) and sys.version_info < (3, 0, 0):
# https://www.python.org/dev/peps/pep-0476/
# Look for 'Opting out' section in this that talks about disabling the certificate verification
# Following line helps to disable globally
ssl._create_default_https_context = ssl._create_unverified_context
# Disabling the security warning message, as the certificate verification is disabled
urllib3.disable_warnings()
try:
service_instance = connect.SmartConnect(
host=self.params['host'],
user=self.params['username'],
pwd=self.params['password'],
port=int(self.params['port']))
atexit.register(connect.Disconnect, service_instance)
self.service_instance = service_instance
self._cache_metrics_metadata(self.params['host'])
except KeyError as ke:
util.sendEvent(
"Plugin vmware: Key Error",
"Improper param.json, key missing: [" + str(ke) + "]", "error")
sys.exit(-1)
except ConnectionError as ce:
util.sendEvent(
"Plugin vmware: Error connecting to vCenter",
"Could not connect to the specified vCenter host: [" +
str(ce) + "]", "critical")
sys.exit(-1)
except StandardError as se:
util.sendEvent("Plugin vmware: Unknown Error", "[" + str(se) + "]",
"critical")
sys.exit(-1)
except vim.fault.InvalidLogin as il:
util.sendEvent(
"Plugin vmware: Error logging into vCenter",
"Could not login to the specified vCenter host: [" + str(il) +
"]", "critical")
sys.exit(-1)
import datetime
import json
import logging
import time
import traceback
from collections import namedtuple
from requests import Session
from requests.packages import urllib3
urllib3.disable_warnings(
) # for self-signed certificates- still safe (just stops verbal warnings)
class BadRequest(Exception):
pass
class AuthenticationError(Exception):
pass
class NotFoundError(Exception):
pass
class ServerError(Exception):
pass
class OtherError(Exception):
import pandas as pd
from xpinyin import Pinyin
from idlelib.iomenu import encoding
import getpass
import time
from netaddr import IPNetwork, IPAddress
# Checl IP address
#if IPAddress("192.168.0.1") in IPNetwork("192.168.0.0/24"):
# print "Yay!"
vsipname = input('VIP Address: ')
print("")
vsipname = vsipname.strip()
#srcip = IPAddress(vsipname)
p = Pinyin()
urllib3.disable_warnings() #disable ssl warning
#導入excel數據 ------begin----------
config_file = input("請輸入需要導入的excel表格名稱，請帶尾碼如：配置資訊.xlsx，設定檔請放在python腳本一個資料夾下:")
#data = xlrd.open_workbook(config_file,'r')
#data = pd.read_excel(config_file, sheet_name="sheet1", engine='openpyxl')
data = pd.read_excel(config_file,
sheet_name=0,
engine='openpyxl',
keep_default_na=False)
#data = pd.read_excel(config_file, sheet_name=0, engine='xlrd')
#data = pd.read_excel(config_file, sheet_name=0)
#df=pandas.read_excel(‘data.xlsx’,engine=‘openpyxl’)
print(data)
#table = data.sheets()[0]
#table = data
def __init__(self):
disable_warnings()
lf = LoggerFactory()
self.logger = lf.create_logger()
def rest_call(self, url, method, data=None, jsondata=False, sensitive=False, extraheaders=None, timeout=60,
retry=None, max_retry=30, retry_sleep=10):
"""
Generic REST call worker function
:param url: URL for the REST call
:param method: METHOD for the REST call
:param data: Optional DATA for the call (for POST/PUT/etc.)
:param jsondata: If data should be sent as JSON and not www-form-urlencoded
:param sensitive: Flag if content request/response should be hidden from logging functions
:param extraheaders: Extra/modified headers
:param timeout: Requests Timeout
:param retry: Boolean if request should be retried if failure.
:param max_retry: Maximum number of retries before giving up
:param retry_sleep: Time inbetween retries.
:return: Tuple (Boolean success or failure, Requests.Response object)
"""
logger.debug('rest_call:')
# check for SSL verification on this session
verify = self.__ap_verifyssl
# Retry loop counter
retry_count = 0
if not extraheaders:
extraheaders = {}
# Run once logic.
if not retry:
run_once = True
else:
run_once = False
if jsondata:
# need to make sure data is cast to JSON.
data = json.dumps(data)
extraheaders['Content-Type'] = 'application/json'
while retry or run_once:
headers = {'Accept': 'application/json'}
# if the request needs extra headers, add them.
if extraheaders and type(extraheaders) is dict:
for key, value in extraheaders.items():
headers[key] = value
cookie = self.__http_session.cookies.get_dict()
# disable warnings and verification if requested.
if not verify:
# disable warnings for SSL certs.
urllib3.disable_warnings()
logger.debug('url = {0}'.format(url))
# make request
try:
if data:
# pre request, dump simple JSON debug
if not sensitive and (logger_level <= logging.DEBUG and logger_level != logging.NOTSET):
logger.debug('\n\tREQUEST: %s %s\n\tHEADERS: %s\n\tCOOKIES: %s\n\tDATA: %s\n',
method.upper(), url, headers, cookie, data)
response = getattr(self.__http_session, method)(url, data=data, headers=headers, verify=verify,
stream=True, timeout=timeout, allow_redirects=False)
else:
# pre request, dump simple JSON debug
if not sensitive and (logger_level <= logging.DEBUG and logger_level != logging.NOTSET):
logger.debug('\n\tREQUEST: %s %s\n\tHEADERS: %s\n\tCOOKIES: %s\n',
method.upper(), url, headers, cookie)
response = getattr(self.__http_session, method)(url, headers=headers, verify=verify, stream=True,
timeout=timeout, allow_redirects=False)
# if it's a non-good response, don't accept it - wait and retry
if response.status_code not in [requests.codes.ok,
requests.codes.no_content,
requests.codes.found,
requests.codes.moved]:
# Simple JSON debug
if not sensitive and (logger_level <= logging.DEBUG and logger_level != logging.NOTSET):
try:
logger.debug('RESPONSE HEADERS: %s\n', json.dumps(
json.loads(str(response.headers)), indent=4))
except ValueError:
logger.debug('RESPONSE HEADERS: %s\n', str(response.headers))
try:
logger.debug('RESPONSE: %s\n', json.dumps(response.json(), indent=4))
except ValueError:
logger.debug('RESPONSE: %s\n', str(response.text))
logger.debug("Error, non-200 response received: %s", response.status_code)
if retry:
# keep retrying
retry_count += 1
if retry_count >= max_retry:
logger.info("Max retries of %s reached.", max_retry)
retry = False
# wait a bit to see if issue clears.
sleep(retry_sleep)
else:
# run once is over.
run_once = False
return False, response
else:
# Simple JSON debug
if not sensitive and (logger_level <= logging.DEBUG and logger_level != logging.NOTSET):
try:
logger.debug('RESPONSE HEADERS: %s\n', json.dumps(
json.loads(str(response.headers)), indent=4))
logger.debug('RESPONSE: %s\n', json.dumps(response.json(), indent=4))
except ValueError:
logger.debug('RESPONSE HEADERS: %s\n', str(response.headers))
logger.debug('RESPONSE: %s\n', str(response.text))
# if retries have been done, update log if requested.
if retry_count > 0:
logger.debug("Got good response after %s retries. ", retry_count)
# run once is over, if set.
run_once = False
return True, response
except requests.exceptions.Timeout:
logger.info("Error, request timeout reached.")
if retry:
# keep retrying
retry_count += 1
if retry_count >= max_retry:
logger.info("Max retries of %s reached.", max_retry)
retry = False
# wait a bit to see if issue clears.
sleep(retry_sleep)
else:
# run once is over.
# run_once = False
return False, None
def __init__(self,
url=None,
ssl_verify=True,
accept="application/json, */*",
content_type="application/json",
user_agent="RESTinstance/%s" % (__version__),
proxies={},
schema={},
spec={},
instances=[],
loglevel="WARN"):
self.request = {
"method": None,
"url": None,
"scheme": "",
"netloc": "",
"path": "",
"query": {},
"body": None,
"data": None,
"auth": None,
"headers": {
"Accept": REST._input_string(accept),
"Content-Type": REST._input_string(content_type),
"User-Agent": REST._input_string(user_agent),
},
"proxies": REST._input_object(proxies),
"timeout": [None, None],
"cert": None,
"sslVerify": REST._input_ssl_verify(ssl_verify),
"allowRedirects": True,
}
if url:
url = REST._input_string(url)
if url.endswith("/"):
url = url[:-1]
if not url.startswith(("http://", "https://")):
url = "http://" + url
url_parts = urlparse(url)
self.request["scheme"] = url_parts.scheme
self.request["netloc"] = url_parts.netloc
self.request["path"] = url_parts.path
if not self.request["sslVerify"]:
disable_warnings()
self.schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": url,
"description": None,
"default": True,
"examples": [],
"type": "object",
"properties": {
"request": {
"type": "object",
"properties": {}
},
"response": {
"type": "object",
"properties": {}
},
},
}
self.schema.update(self._input_object(schema))
self.spec = {}
self.spec.update(self._input_object(spec))
self.instances = self._input_array(instances)
self.log_level = self._input_log_level(loglevel)
def __init__(self, timeout=None, clear_handler=None, refresh_handler=None):
"""
Initialize the Docker wrapper.
:param timeout: int
:param clear_handler: callable
:param refresh_handler: callable
"""
assert callable(clear_handler)
assert callable(refresh_handler)
self.exception = None
self.handlers = {
'attach': (self.attach, 'Attach to a running container.'),
'build': (self.build, ("Build a new image from the source"
" code")),
'clear': (clear_handler, "Clear the window."),
'create': (self.create, 'Create a new container.'),
'exec': (self.execute, ("Run a command in a running"
" container.")),
'help': (self.help, "Help on available commands."),
'pause': (self.pause, "Pause all processes within a container."),
'ps': (self.containers, "List containers."),
'port': (self.port, ("List port mappings for the container, or "
"lookup the public-facing port that is "
"NAT-ed to the private_port.")),
'pull': (self.pull, ("Pull an image or a repository from the "
"registry.")),
'push': (self.push, ("Push an image or a repository to the "
"registry.")),
'images': (self.images, "List images."),
'info': (self.info, "Display system-wide information."),
'inspect':
(self.inspect,
"Return low-level information on a " + "container or image."),
'login': (self.login, ("Register or log in to a Docker registry "
"server (defaut "
"\"https://index.docker.io/v1/\").")),
'logs': (self.logs, "Fetch the logs of a container."),
'refresh': (refresh_handler, "Refresh autocompletions."),
'restart': (self.restart, "Restart a running container."),
'run': (self.run, "Run a command in a new container."),
'rm': (self.rm, "Remove one or more containers."),
'rmi': (self.rmi, "Remove one or more images."),
'search': (self.search, "Search the Docker Hub for images."),
'shell': (self.shell, "Get shell into a running container."),
'start': (self.start, "Restart a stopped container."),
'stop': (self.stop, "Stop a running container."),
'tag': (self.tag, "Tag an image into a repository."),
'top': (self.top, "Display the running processes of a container."),
'unpause': (self.unpause, ("Unpause all processes within a "
"container.")),
'version': (self.version, "Show the Docker version information."),
'volume create': (self.volume_create, "Create a new volume."),
'volume inspect': (self.volume_inspect, "Inspect one or more "
"volumes."),
'volume ls': (self.volume_ls, "List volumes."),
'volume rm': (self.volume_rm, "Remove a volume."),
}
self.output = None
self.after = None
self.command = None
self.logs = None
self.is_refresh_containers = False
self.is_refresh_running = False
self.is_refresh_images = False
self.is_refresh_volumes = False
disable_warnings()
if sys.platform.startswith('darwin') \
or sys.platform.startswith('win32'):
try:
# mac or win
kwargs = kwargs_from_env()
# hack from here:
# http://docker-py.readthedocs.org/en/latest/boot2docker/
# See also: https://github.com/docker/docker-py/issues/406
if 'tls' in kwargs:
kwargs['tls'].assert_hostname = False
kwargs['timeout'] = timeout
self.instance = AutoVersionClient(**kwargs)
except DockerException as x:
if 'CERTIFICATE_VERIFY_FAILED' in x.message:
raise DockerSslException(x)
elif 'ConnectTimeoutError' in x.message:
raise DockerTimeoutException(x)
else:
raise x
else:
# unix-based
kwargs = kwargs_from_env(ssl_version=ssl.PROTOCOL_TLSv1,
assert_hostname=False)
kwargs['timeout'] = timeout
self.instance = AutoVersionClient(**kwargs)
# ================================================================================================
# local imports
from .auth import AuthConfig
from .resource import Resource, JSONResource
from .exception import RestClientConfigurationError
from .utils import URLValidator
# ================================================================================================
# Interface tweak
from requests.packages.urllib3 import disable_warnings
from requests.packages.urllib3.exceptions import InsecureRequestWarning
logger = logging.getLogger(__name__)
disable_warnings(InsecureRequestWarning)
class ParameterConfig:
"""Contain and validate parameters for a REST endpoint. As this is a
configuration container only, the main purpose is to store the config and
check if the input aligns with the intended use. There is no validation
beyond this point
"""
# -----------------------------------------------------------------------------------------------------
def __init__(self,
name: str,
required: bool = False,
multiple: bool = False,
from argparse import ArgumentParser
from json import dumps, loads
from os.path import basename, splitext
from pprint import pprint
from sys import stderr, path
from time import time
from requests import post
from requests.packages.urllib3 import disable_warnings
from urllib3.exceptions import InsecureRequestWarning
# Import encryption routines.
path.insert(1, 'RAU_crypto')
from RAU_crypto import RAUCipher
disable_warnings(category=InsecureRequestWarning)
def send_request(url, files):
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Connection': 'close',
'Accept-Language': 'en-US,en;q=0.5',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Upgrade-Insecure-Requests': '1',
}
response = post(url,
files=files,
headers=headers,
