def _get_status(self, url):
data = None
# verify that the URL is valid
try:
six.moves.urllib.parse.urlparse(url)
except Exception:
print(
"The URL configured for requesting the status page appears to be invalid. Please verify that the URL is correct in your monitor configuration. The specified url: %s"
% url
)
return data
# attempt to request server status
try:
request = six.moves.urllib.request.Request(self._monitor_url)
if self._monitor_user is not None:
b64cred = base64.encodestring(
"%s:%s" % (self._monitor_user, self._monitor_password)
).replace("\n", "")
request.add_header("Authorization", "Basic %s" % b64cred)
opener = six.moves.urllib.request.build_opener(BindableHTTPHandler)
handle = opener.open(request)
data = handle.read()
except six.moves.urllib.error.HTTPError as err:
message = (
"An HTTP error occurred attempting to retrieve the status. Please consult your server logs to determine the cause. HTTP error code: %s",
err.code,
)
if err.code == 404:
message = "The URL used to request the status page appears to be incorrect. Please verify the correct URL and update your apache_monitor configuration."
elif err.code == 403:
message = "The server is denying access to the URL specified for requesting the status page. Please verify that permissions to access the status page are correctly configured in your server configuration and that your apache_monitor configuration reflects the same configuration requirements."
elif err.code >= 500 or err.code < 600:
message = (
"The server failed to fulfill the request to get the status page. Please consult your server logs to determine the cause. HTTP error code: %s",
err.code,
)
self._logger.error(message)
data = None
except six.moves.urllib.error.URLError as err:
message = (
"The was an error attempting to reach the server. Make sure the server is running and properly configured. The error reported is: %s"
% str(err)
)
if err.reason.errno == 111:
message = (
"The HTTP server does not appear to running or cannot be reached. Please check that it is running and is reachable at the address: %s"
% url.netloc
)
self._logger.error(message)
data = None
except Exception as e:
self._logger.error(
"An error occurred attempting to request the server status: %s" % e
)
data = None
return data
def _add_auth_header(request, target):
if 'username' in target:
u, p = target.get('username'), target.get('password', '')
request.add_header('Authorization', basic_auth_header(u, p))
else: # try netrc
try:
host = urlparse(target['url']).hostname
a = netrc.netrc().authenticators(host)
request.add_header('Authorization', basic_auth_header(a[0], a[2]))
except (netrc.NetrcParseError, IOError, TypeError):
pass
def download_proxies(path=PROXY_PATH):
# Downloading without proxy
opener = six.moves.urllib.request.build_opener(six.moves.urllib.request.ProxyHandler())
six.moves.urllib.request.install_opener(opener)
request = six.moves.urllib.request.Request('http://www.ip-adress.com/proxy_list/')
request.add_header('user-agent',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36')
request.add_header('referer', 'htpps://www.google.com/')
f = six.moves.urllib.request.urlopen(request)
pattern = r'\d*\.\d*\.\d*\.\d*\</a>:\d*'
found = [i.replace('</a>', '') + '\n' for i in re.findall(pattern, f.read())]
dump_proxies_to_file(found[:20], path) # 20 top proxies
def submit(filename, task_name, token, contest_url, open_webbrowser):
if 'contest-url' not in configuration:
print(
"Configuration file could not be loaded. "
+ "Please run submit.py --configure "
+ "to configure the application.",
file=sys.stderr,
)
return 1
print('oioioi-submit', file=sys.stderr)
print('=============', file=sys.stderr)
try:
if not contest_url.endswith('/'):
contest_url += '/'
if not task_name:
task_name, _ = splitext(filename)
with open(filename, 'rb') as solution_file:
form = MultiPartForm()
form.add_field('token', token)
form.add_field('task', task_name)
form.add_file('file', solution_file.name, filehandle=solution_file)
body = str(form)
request = six.moves.urllib.request.Request(
'%ssubmitservice/submit/' % contest_url
)
request.add_header('Content-Type', form.get_content_type())
request.add_header('Content-Length', len(body))
request.add_data(body)
result = json.loads(six.moves.request.urlopen(request).read())
base_url = six.moves.urllib.parse.urlparse(contest_url)
if 'error_code' not in result:
result_url = '%s://%s%s' % (
base_url.scheme,
base_url.netloc,
result['result_url'],
)
print("Submission succeeded! View your status at:", file=sys.stderr)
print(result_url, file=sys.stderr)
if open_webbrowser:
webbrowser.open_new_tab(result_url)
else:
raise RuntimeError(
error_code_map[result['error_code']] % {'data': result['error_data']}
)
except Exception as e:
print("Error:", e, file=sys.stderr)
print("Submission failed.", file=sys.stderr)
return 1
return 0
def build_request(self):
"""
Builds the HTTP request based on the request URL, HTTP headers and method
@return: Request object
"""
request = six.moves.urllib.request.Request(self.url,
data=self.request_data)
if self.request_headers:
for header in self.request_headers:
request.add_header(header["header"], header["value"])
# seems awkward to override the GET method, but internally it flips
# between GET and POST anyway based on the existence of request body
request.get_method = lambda: self.request_method
return request
def _get_token_request(
token,
method,
uri,
data=None): # type: (unicode, unicode, unicode, Any) -> Any
"""
:param token:
:param method:
:param uri:
:param data:
:return:
"""
logger.debug("_get_token_request('%s', '%s', %s)", method, uri, data)
request = _get_request(method, uri, data)
request.add_header("token", token)
return request
def submit(filename, task_name, token, contest_url, open_webbrowser):
if 'contest-url' not in configuration:
print("Configuration file could not be loaded. " +
"Please run submit.py --configure " +
"to configure the application.", file=sys.stderr)
return 1
print('oioioi-submit', file=sys.stderr)
print('=============', file=sys.stderr)
try:
if not contest_url.endswith('/'):
contest_url += '/'
if not task_name:
task_name, _ = splitext(filename)
with open(filename, 'rb') as solution_file:
form = MultiPartForm()
form.add_field('token', token)
form.add_field('task', task_name)
form.add_file('file', solution_file.name, filehandle=solution_file)
body = str(form)
request = six.moves.urllib.request.Request('%ssubmitservice/submit/' % contest_url)
request.add_header('Content-Type', form.get_content_type())
request.add_header('Content-Length', len(body))
request.add_data(body)
result = json.loads(six.moves.request.urlopen(request).read())
base_url = six.moves.urllib.parse.urlparse(contest_url)
if 'error_code' not in result:
result_url = '%s://%s%s' % (base_url.scheme, base_url.netloc,
result['result_url'])
print("Submission succeeded! View your status at:", file=sys.stderr)
print(result_url, file=sys.stderr)
if open_webbrowser:
webbrowser.open_new_tab(result_url)
else:
raise RuntimeError(error_code_map[result['error_code']] %
{'data': result['error_data']})
except Exception as e:
print("Error:", e, file=sys.stderr)
print("Submission failed.", file=sys.stderr)
return 1
return 0
def http_request(self, request):
request.add_header("Accept-Encoding", "gzip")
for header in request.headers:
if header.lower() == "user-agent":
request.headers[header] += " gzip"
return request
def _FetchDiscoveryDoc(config, doc_format):
"""Fetch discovery documents generated from a cloud service.
Args:
config: An API config.
doc_format: The requested format for the discovery doc. (rest|rpc)
Raises:
ServerRequestException: If fetching the generated discovery doc fails.
Returns:
A list of discovery doc strings.
"""
body = json.dumps({'config': config}, indent=2, sort_keys=True)
request = six.moves.urllib.request.Request(DISCOVERY_DOC_BASE + doc_format,
body)
request.add_header('content-type', 'application/json')
try:
with contextlib.closing(
six.moves.urllib.request.urlopen(request)) as response:
return response.read()
except six.moves.urllib.error.HTTPError as error:
raise ServerRequestException(error)
def make_fetch_call(rpc,
url,
payload=None,
method=GET,
headers={},
allow_truncated=False,
follow_redirects=True,
validate_certificate=None):
"""Executes the RPC call to fetch a given HTTP URL.
The first argument is a UserRPC instance. See urlfetch.fetch for a
thorough description of remaining arguments.
Raises:
InvalidMethodError: if requested method is not in _VALID_METHODS
ResponseTooLargeError: if the response payload is too large
InvalidURLError: if there are issues with the content/size of the
requested URL
Returns:
The rpc object passed into the function.
"""
assert rpc.service == 'urlfetch', repr(rpc.service)
if isinstance(method, six.string_types):
method = method.upper()
method = _URL_STRING_MAP.get(method, method)
if method not in _VALID_METHODS:
raise InvalidMethodError('Invalid method %s.' % str(method))
if _is_fetching_self(url, method):
raise InvalidURLError(
"App cannot fetch the same URL as the one used for "
"the request.")
request = urlfetch_service_pb.URLFetchRequest()
response = urlfetch_service_pb.URLFetchResponse()
if isinstance(url, six.text_type):
url = url.encode('UTF-8')
request.set_url(url)
if method == GET:
request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
elif method == POST:
request.set_method(urlfetch_service_pb.URLFetchRequest.POST)
elif method == HEAD:
request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD)
elif method == PUT:
request.set_method(urlfetch_service_pb.URLFetchRequest.PUT)
elif method == DELETE:
request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE)
elif method == PATCH:
request.set_method(urlfetch_service_pb.URLFetchRequest.PATCH)
if payload and method in (POST, PUT, PATCH):
request.set_payload(payload)
for key, value in six.iteritems(headers):
header_proto = request.add_header()
header_proto.set_key(key)
header_proto.set_value(str(value))
request.set_followredirects(follow_redirects)
if validate_certificate is not None:
request.set_mustvalidateservercertificate(validate_certificate)
if rpc.deadline is not None:
request.set_deadline(rpc.deadline)
rpc.make_call('Fetch', request, response, _get_fetch_result,
allow_truncated)
return rpc
def _get_request(method,
uri,
token=None,
post_data=None
): # type: (unicode, unicode, Optional[unicode], Any) -> Any
"""
:param method:
:param uri:
:param post_data:
:return:
"""
logger.debug("_get_request('%s', '%s', '%s', %s)", method, uri, token,
post_data)
json_data = None
if post_data is not None:
json_data = jsonify(post_data)
request = six.moves.urllib.request.Request(url=_base_url + _base_uri + uri,
data=json_data)
request.get_method = lambda: method
if json_data is not None:
request.add_header("Content-Length", len(json_data))
request.add_header("Content-Type", "application/json; charset=utf-8")
request.add_header("User-Agent",
"sd2xmltv/0.2.1 ([email protected])")
request.add_header("Accept", "application/json")
request.add_header("Accept-Encoding", "deflate, gzip")
if token is not None:
request.add_header("token", token)
return request
