def get_form(self, payment, data=None):
if payment.status == 'waiting':
payment.change_status('input')
form = DummyForm(data=data,
hidden_inputs=False,
provider=self,
payment=payment)
if form.is_valid():
new_status = form.cleaned_data['status']
payment.change_status(new_status)
new_fraud_status = form.cleaned_data['fraud_status']
payment.change_fraud_status(new_fraud_status)
gateway_response = form.cleaned_data.get('gateway_response')
verification_result = form.cleaned_data.get('verification_result')
if gateway_response or verification_result:
if gateway_response == '3ds-disabled':
# Standard request without 3DSecure
pass
elif gateway_response == '3ds-redirect':
# Simulate redirect to 3DS and get back to normal
# payment processing
process_url = payment.get_process_url()
params = urlencode(
{'verification_result': verification_result})
redirect_url = '%s?%s' % (process_url, params)
raise RedirectNeeded(redirect_url)
elif gateway_response == 'failure':
# Gateway raises error (HTTP 500 for example)
raise URLError('Opps')
elif gateway_response == 'payment-error':
raise PaymentError('Unsupported operation')
if new_status in ['preauth', 'confirmed']:
raise RedirectNeeded(payment.get_success_url())
raise RedirectNeeded(payment.get_failure_url())
return form
def test_installs(self, _extract, _mkdir, _check_hash):
self.fh.download = MagicMock()
for url in self.valid_urls:
filename = urlparse(url).path
dest = os.path.join('foo', 'fetched', os.path.basename(filename))
_extract.return_value = dest
with patch.dict('os.environ', {'CHARM_DIR': 'foo'}):
where = self.fh.install(url, checksum='deadbeef')
self.fh.download.assert_called_with(url, dest)
_extract.assert_called_with(dest, None)
_check_hash.assert_called_with(dest, 'deadbeef', 'sha1')
self.assertEqual(where, dest)
_check_hash.reset_mock()
url = "http://www.example.com/archive.tar.gz"
self.fh.download.side_effect = URLError('fail')
with patch.dict('os.environ', {'CHARM_DIR': 'foo'}):
self.assertRaises(UnhandledSource, self.fh.install, url)
self.fh.download.side_effect = OSError('fail')
with patch.dict('os.environ', {'CHARM_DIR': 'foo'}):
self.assertRaises(UnhandledSource, self.fh.install, url)
def test_request_http_error_retry(self):
mocked_response = mock.MagicMock(status=200, spec=HTTPResponse)
mocked_response.read.return_value = b'{ "foo": "bar" }'
mocked_response.headers = mock.Mock()
mocked_response.headers.get_charset.return_value = 'utf-8'
# raise error on first call, then a valid response
mock_side_effect = [URLError('test error'), mocked_response]
test_servers = deque(TEST_SERVERS)
http_client = ElasticsearchRequestController(
test_servers, TEST_TIMEOUT, None, False, self._mocked_logger)
# pre-flight check, basically cannot fail but won't hurt
self.assertEqual(http_client._servers[0], TEST_SERVER_1)
# test
with mock.patch.object(http_client, '_url_opener') as mock_url_opener:
mock_url_opener.open.side_effect = mock_side_effect
result = http_client.request('/', data=None)
# http_client._servers[0] is a deque and has been rotated after the first error,
# so now we expect the second server item to be at the first index
self.assertEqual(http_client._servers[0], TEST_SERVER_2)
# the response should match in any way, so check it
self.assertEqual(result, dict(foo='bar'))
def get_all_comments(self):
"""使用多线程抓取评论页"""
# 当当网要重写这个函数
self.build_queue()
if self.urlQueue.qsize() > 10:
self.use_proxy = True
if not self.urlQueue.empty():
threads = []
# 可以调节线程数， 进而控制抓取速度
thread_num = self.urlQueue.qsize() // 3 + 1
if thread_num >= 15:
thread_num = 15
if thread_num <= 3:
thread_num = 3
for i in range(thread_num):
t = Thread(target=self.get_comments)
threads.append(t)
for t in threads:
t.start()
for t in threads:
# 多线程多join的情况下，依次执行各线程的join方法, 这样可以确保主线程最后退出， 且各个线程间没有阻塞
t.join()
else:
raise URLError('商品链接地址不能被识别')
def test_open_auklet_url(self):
url = self.client.base_url + "private/devices/config/"
with patch('auklet.utils.urlopen') as url_open:
self.assertIsNotNone(open_auklet_url(url, self.client.apikey))
url_open.side_effect = HTTPError(url=None,
code=401,
msg=None,
hdrs=None,
fp=None)
self.assertRaises(AukletConfigurationError,
lambda: open_auklet_url(url, self.client.apikey))
url_open.side_effect = HTTPError(url=None,
code=None,
msg=None,
hdrs=None,
fp=None)
self.assertRaises(HTTPError,
lambda: open_auklet_url(url, self.client.apikey))
url_open.side_effect = URLError("")
self.assertIsNone(open_auklet_url(url, self.client.apikey))
def parse_redis_url(url):
from urllib.parse import urlparse, parse_qs
from urllib.error import URLError
parsed = urlparse(url)
if parsed.scheme != 'redis':
raise URLError('Redis URL does not have the redis scheme')
path = parsed.path[1:] or ''
query = parse_qs(parsed.query or '')
if path:
db = int(path)
elif 'db' in query:
db = int(query['db'][0])
else:
db = 0
options = {'host': parsed.hostname, 'port': parsed.port, 'db': db}
if parsed.password:
options['password'] = parsed.password
return options
def _catch_error_request(self, url='', data=None, retrieve=False):
url = url or self.url
try:
response = self.session.open(url, data, 5)
# checking that tracker is'nt blocked
if not response.geturl().startswith((self.url, self.url_dl)):
raise URLError(f"{self.url} is blocked. Try another proxy.")
except (socket.error, socket.timeout) as err:
if not retrieve:
return self._catch_error_request(url, data, True)
logger.error(err)
self.error = f"{self.url} is not response! Maybe it is blocked."
if "no host given" in err.args:
self.error = "Proxy is bad, try another!"
except (URLError, HTTPError) as err:
logger.error(err.reason)
self.error = err.reason
if hasattr(err, 'code'):
self.error = f"Request to {url} failed with status: {err.code}"
else:
return response
return None
def download_url_resource(path, output=None):
"""
Télécharger un fichier à une URL et renvoyer le chemin du fichier local téléchargé
:param output: Chemin de sortie, fichier dans le répertoire temporaire si None
:type output: str | None
"""
if output and os.path.exists(output):
logging.warning(
_("The download destination file at {path} already exists. Skipped."
).format(path=output))
return output
try:
resource = get_url_resource(path, False, stream=True)
except (IOError, OSError):
raise URLError(
"The resource at {path} cannot be downloaded.".format(path=path),
path)
resource_file = NamedTemporaryFile(
delete=False) if output is None else open(output, 'wb')
for chunk in resource.iter_content(16384):
resource_file.write(chunk)
resource_file.close()
return resource_file.name
def load(url,
data: bytes = None,
timeout: float = socket._GLOBAL_DEFAULT_TIMEOUT,
cafile: str = None,
capath: str = None,
cadefault: bool = False,
context: ssl.SSLContext = None):
if isinstance(url, urllib.request.Request):
request = url
elif isinstance(url, str):
request = Request(url)
else:
raise URLError("Unvalid URL")
request.add_header("Accept-encoding", "gzip")
http_cli = __HTTP_CLIENT_CLASS__(request=request,
data=data,
timeout=timeout,
cafile=cafile,
capath=capath,
cadefault=cadefault,
context=context)
res = http_cli.urlopen()
txt = http_cli.read_response_body(res)
return txt, res
def buildBingoHintList(boardURL):
try:
if len(boardURL) > 256:
raise URLError(f"URL too large {len(boardURL)}")
with urllib.request.urlopen(boardURL + "/board") as board:
if board.length and 0 < board.length < 4096:
goalList = board.read()
else:
raise HTTPError(f"Board of invalid size {board.length}")
except (URLError, HTTPError) as e:
logger = logging.getLogger('')
logger.info(f"Could not retrieve board info. Using default bingo hints instead: {e}")
genericBingo = read_json(data_path('Bingo/generic_bingo_hints.json'))
return genericBingo['settings']['item_hints']
# Goal list returned from Bingosync is a sequential list of all of the goals on the bingo board, starting at top-left and moving to the right.
# Each goal is a dictionary with attributes for name, slot, and colours. The only one we use is the name
goalList = [goal['name'] for goal in json.loads(goalList)]
goalHintRequirements = read_json(data_path('Bingo/bingo_goals.json'))
hintsToAdd = {}
for goal in goalList:
# Using 'get' here ensures some level of forward compatibility, where new goals added to randomiser bingo won't
# cause the generator to crash (though those hints won't have item hints for them)
requirements = goalHintRequirements.get(goal,{})
if len(requirements) != 0:
for item in requirements:
hintsToAdd[item] = max(hintsToAdd.get(item, 0), requirements[item]['count'])
# Items to be hinted need to be included in the item_hints list once for each instance you want hinted
# (e.g. if you want all three strength upgrades to be hintes it needs to be in the list three times)
hints = []
for key, value in hintsToAdd.items():
for _ in range(value):
hints.append(key)
return hints
def _cache(self, path):
"""Cache the file specified by path.
Creates a copy of the file in the datasource cache.
"""
# We import these here because importing urllib2 is slow and
# a significant fraction of numpy's total import time.
if sys.version_info[0] >= 3:
from urllib.request import urlopen
from urllib.error import URLError
else:
from urllib2 import urlopen
from urllib2 import URLError
upath = self.abspath(path)
# ensure directory exists
if not os.path.exists(os.path.dirname(upath)):
os.makedirs(os.path.dirname(upath))
# TODO: Doesn't handle compressed files!
if self._isurl(path):
try:
openedurl = urlopen(path)
f = _open(upath, 'wb')
try:
shutil.copyfileobj(openedurl, f)
finally:
f.close()
openedurl.close()
except URLError:
raise URLError("URL not found: %s" % path)
else:
shutil.copyfile(path, upath)
return upath
def _get_swagger_spec(self, endpoint_name: str) -> Spec:
"""Get Swagger spec of specified service."""
logic_module = self._get_logic_module(endpoint_name)
schema_url = utils.get_swagger_url_by_logic_module(logic_module)
if schema_url not in self._specs:
try:
# Use stored specification of the module
spec_dict = logic_module.api_specification
# Pull specification of the module from its service and store it
if spec_dict is None:
response = utils.get_swagger_from_url(schema_url)
spec_dict = response.json()
logic_module.api_specification = spec_dict
logic_module.save()
except URLError:
raise URLError(f'Make sure that {schema_url} is accessible.')
swagger_spec = Spec.from_dict(spec_dict, config=self.SWAGGER_CONFIG)
self._specs[schema_url] = swagger_spec
return self._specs[schema_url]
def test_network_loss(self):
""" Updates should be sent when the network is up again. """
filename = "networkless file.txt"
doc_id = self.remote.make_file_with_blob("/", filename,
b"Initial content.")
# Download file
local_path = f"/{doc_id}_file-content/{filename}"
with patch.object(self.manager_1,
"open_local_file",
new=open_local_file):
self.direct_edit._prepare_edit(self.nuxeo_url, doc_id)
self.wait_sync(timeout=2, fail_if_timeout=False)
# Simulate server error
bad_remote = self.get_bad_remote()
error = URLError(
"[Errno 10051] (Mock) socket operation was attempted to an unreachable network"
)
bad_remote.make_upload_raise(error)
with patch.object(self.engine_1, "remote", new=bad_remote):
# Update file content
self.local.update_content(local_path, b"Updated")
time.sleep(5)
self.local.update_content(local_path, b"Updated twice")
time.sleep(5)
# The file should not be updated on the server
assert (self.remote.get_blob(
self.remote.get_info(doc_id)) == b"Initial content.")
# Check the file is reuploaded when the network come again
self.wait_sync(timeout=12, fail_if_timeout=False)
assert (self.remote.get_blob(
self.remote.get_info(doc_id)) == b"Updated twice")
def test_url(url, retry=2, retry_period=1, timeout=10):
request = Request(url)
request.get_method = lambda: 'HEAD'
try:
response = urlopen(request)
except HTTPError as e:
if e.code == 503 and retry:
time.sleep(retry_period)
return test_url(url,
retry=retry - 1,
retry_period=retry_period,
timeout=timeout)
e.msg += ' (%s)' % url
raise
except URLError as e:
if isinstance(e.reason, socket.timeout) and retry:
time.sleep(retry_period)
return test_url(url,
retry=retry - 1,
retry_period=retry_period,
timeout=timeout)
raise URLError(str(e) + ' (%s)' % url)
return response
def main(args=None):
ARGS = parse_args(args if len(args) > 0 else sys.argv[1:])
logging.basicConfig(level=logging.INFO)
# value at first index is of current subreddit, second index is total
TOTAL, DOWNLOADED, ERRORS, SKIPPED, FAILED = [0, 0], [0,
0], [0,
0], [0,
0], [0, 0]
PROG_REPORT = [TOTAL, DOWNLOADED, ERRORS, SKIPPED, FAILED]
# Create the specified directory if it doesn't already exist.
if not pathexists(ARGS.dir):
mkdir(ARGS.dir)
# If a regex has been specified, compile the rule (once)
RE_RULE = None
if ARGS.regex:
RE_RULE = re.compile(ARGS.regex)
# compile reddit comment url to check if url is one of them
reddit_comment_regex = re.compile(r'.*reddit\.com\/r\/(.*?)\/comments')
LAST = ARGS.last
start_time = None
ITEM = None
sort_type = ARGS.sort_type
if sort_type:
sort_type = sort_type.lower()
# check to see if ARGS.subreddit is subreddit or subreddit-list
if os.path.isfile(
ARGS.subreddit) and os.path.splitext(ARGS.subreddit)[1] != '':
ARGS.subreddit_list = ARGS.subreddit
if ARGS.subreddit_list:
# ARGS.subreddit_list = ARGS.subreddit_list[0] # can't remember why I did this -jtara1
subreddit_file = ARGS.subreddit_list
subreddit_list = parse_subreddit_list(subreddit_file, ARGS.dir)
if ARGS.verbose:
print('subreddit_list = %s' % subreddit_list)
elif not ARGS.subreddit_list:
subreddit_list = [(ARGS.subreddit, ARGS.dir)]
# file used to store last reddit id
log_file = '._history.txt'
# iterate through subreddit(s)
for index, section in enumerate(subreddit_list):
(ARGS.subreddit, ARGS.dir) = section
FINISHED = False
if ARGS.verbose:
print('index: %s, %s, %s' % (index, ARGS.subreddit, ARGS.dir))
# load last_id or create new entry for last_id in log_data
log_data, last_id = process_subreddit_last_id(ARGS.subreddit,
ARGS.sort_type, ARGS.dir,
log_file, ARGS.dir)
if ARGS.restart:
last_id = ''
TOTAL[0], DOWNLOADED[0], ERRORS[0], SKIPPED[0], FAILED[
0], FILECOUNT = 0, 0, 0, 0, 0, 0
# ITEMS loop - begin the loop to get reddit submissions & download media from them
while not FINISHED:
if ARGS.verbose:
print()
ITEMS = getitems(ARGS.subreddit,
multireddit=ARGS.multireddit,
previd=last_id,
reddit_sort=sort_type)
# debug ITEMS variable value
# if ARGS.verbose:
# history_log(os.getcwd(), 'ITEMS.txt', 'write', ITEMS)
# measure time and set the program to wait 4 second between request
# as per reddit api guidelines
end_time = time.process_time()
if start_time is not None:
elapsed_time = end_time - start_time
if elapsed_time <= 4: # throttling
time.sleep(4 - elapsed_time)
start_time = time.process_time()
# No more items to process
if not ITEMS:
if ARGS.verbose:
print('No more ITEMS for %s %s' %
(ARGS.subreddit, ARGS.sort_type))
break
for ITEM in ITEMS:
TOTAL[0] += 1
if ('reddit.com/r/' + ARGS.subreddit + '/comments/'
in ITEM['url'] or re.match(reddit_comment_regex,
ITEM['url']) is not None):
# hotfix for when last item is comment submission which caused infinite looping
last_id = ITEM['id'] if ITEM is not None else None
if last_id:
log_data[ARGS.subreddit][
ARGS.sort_type]['last-id'] = last_id
history_log(ARGS.dir,
log_file,
mode='write',
write_data=log_data)
continue
# don't download if url is reddit metrics url
if 'redditmetrics.com' in ITEM['url']:
if ARGS.verbose:
print('\t%s was skipped.' % ITEM['url'])
SKIPPED[0] += 1
continue
if ITEM['score'] < ARGS.score:
if ARGS.verbose:
print(' SCORE: {} has score of {}'.format(
ITEM['id'], ITEM['score']))
'which is lower than required score of {}.'.format(
ARGS.score)
SKIPPED[0] += 1
continue
elif ARGS.sfw and ITEM['over_18']:
if ARGS.verbose:
print(' NSFW: %s is marked as NSFW.' % (ITEM['id']))
SKIPPED[0] += 1
continue
elif ARGS.nsfw and not ITEM['over_18']:
if ARGS.verbose:
print(' Not NSFW, skipping %s' % (ITEM['id']))
SKIPPED[0] += 1
continue
elif ARGS.regex and not re.match(RE_RULE, ITEM['title']):
if ARGS.verbose:
print(' Regex match failed')
SKIPPED[0] += 1
continue
elif ARGS.skipAlbums and 'imgur.com/a/' in ITEM['url']:
if ARGS.verbose:
print(' Album found, skipping %s' % (ITEM['id']))
SKIPPED[0] += 1
continue
if ARGS.title_contain and ARGS.title_contain.lower(
) not in ITEM['title'].lower():
if ARGS.verbose:
print(' Title not contain "{}",'.format(
ARGS.title_contain))
'skipping {}'.format(ITEM['id'])
SKIPPED[0] += 1
continue
try:
URLS = extract_urls(ITEM['url'])
except URLError as e:
print('URLError %s' % e)
continue
except Exception as e:
_log.exception("%s", e)
continue
for URL in URLS:
try:
# Find gfycat if requested
if URL.endswith('gif') and ARGS.mirror_gfycat:
check = gfycat().check(URL)
if check.get("urlKnown"):
URL = check.get('webmUrl')
# Trim any http query off end of file extension.
FILEEXT = pathsplitext(URL)[1]
if '?' in FILEEXT:
FILEEXT = FILEEXT[:FILEEXT.index('?')]
# Only append numbers if more than one file
FILENUM = ('_%d' % FILECOUNT if len(URLS) > 1 else '')
# create filename based on given input from user
if ARGS.filename_format == 'url':
FILENAME = '%s%s%s' % (pathsplitext(
pathbasename(URL))[0], '', FILEEXT)
elif ARGS.filename_format == 'title':
FILENAME = '%s%s%s' % (slugify(
ITEM['title']), FILENUM, FILEEXT)
if len(FILENAME) >= 256:
shortened_item_title = slugify(
ITEM['title'])[:256 - len(FILENAME)]
FILENAME = '%s%s%s' % (shortened_item_title,
FILENUM, FILEEXT)
else:
FILENAME = '%s%s%s' % (ITEM['id'], FILENUM,
FILEEXT)
# join file with directory
FILEPATH = pathjoin(ARGS.dir, FILENAME)
# Improve debuggability list URL before download too.
# url may be wrong so skip that
if URL.encode('utf-8') == 'http://':
raise URLError('Url is empty')
# Download the image
try:
dl = skp = 0
if 'imgur.com' in URL:
fname = os.path.splitext(FILENAME)[0]
save_path = os.path.join(os.getcwd(), ARGS.dir)
downloader = ImgurDownloader(URL,
save_path,
fname,
delete_dne=True,
debug=False)
(dl, skp) = downloader.save_images()
else:
download_from_url(URL, FILEPATH)
dl = 1
# Image downloaded successfully!
if ARGS.verbose:
print('Saved %s as %s' % (URL, FILENAME))
DOWNLOADED[0] += 1
SKIPPED[0] += skp
FILECOUNT += 1
except URLError:
print('We do not support reddituploads links yet'
' skipping....')
except FileExistsException as ERROR:
ERRORS[0] += 1
if ARGS.verbose:
print(ERROR.message)
if ARGS.update:
print(' Update complete, exiting.')
FINISHED = True
break
except ImgurException as e:
ERRORS[0] += 1
except Exception as e:
print(e)
ERRORS[0] += 1
if ARGS.num and (DOWNLOADED[0]) >= ARGS.num:
print(' Download num limit reached, exiting.')
FINISHED = True
break
except WrongFileTypeException as ERROR:
_log_wrongtype(url=URL,
target_dir=ARGS.dir,
filecount=FILECOUNT,
_downloaded=DOWNLOADED[0],
filename=FILENAME)
SKIPPED[0] += 1
except HTTPError as ERROR:
FAILED[0] += 1
except URLError as ERROR:
FAILED[0] += 1
except InvalidURL as ERROR:
FAILED[0] += 1
except Exception as exc:
FAILED[0] += 1
# keep track of last_id id downloaded
last_id = ITEM['id'] if ITEM is not None else None
if last_id:
log_data[ARGS.subreddit][
ARGS.sort_type]['last-id'] = last_id
history_log(ARGS.dir,
log_file,
mode='write',
write_data=log_data)
# break out of URL loop to end of ITEMS loop
if FINISHED:
break
# update variables in PROG_REPORT in SUBREDDIT loop
for var in PROG_REPORT:
var[1] += var[0]
print('Downloaded from %i reddit submissions' % (DOWNLOADED[1]))
print('(Processed %i, Skipped %i, Errors %i)' %
(TOTAL[1], SKIPPED[1], ERRORS[1]))
return DOWNLOADED[1]
def get(self, name, context=None, **kws):
"""See `ITemplateLoader`."""
# Gather some additional information based on the context.
substitutions = {}
if IMailingList.providedBy(context):
mlist = context
domain = context.domain
lookup_contexts = [
mlist.list_id,
mlist.mail_host,
None,
]
substitutions.update(
dict(
list_id=mlist.list_id,
# For backward compatibility, we call this $listname.
listname=mlist.fqdn_listname,
domain_name=domain.mail_host,
language=mlist.preferred_language.code,
))
elif IDomain.providedBy(context):
mlist = None
domain = context
lookup_contexts = [
domain.mail_host,
None,
]
substitutions['domain_name'] = domain.mail_host
elif context is None:
mlist = domain = None
lookup_contexts = [None]
else:
raise ValueError('Bad context type: {!r}'.format(context))
# The passed in keyword arguments take precedence.
substitutions.update(kws)
# See if there's a cached template registered for this name and
# context, passing in the url substitutions. This handles http:,
# https:, and file: urls.
for lookup_context in lookup_contexts:
try:
contents = getUtility(ITemplateManager).get(
name, lookup_context, **substitutions)
except (HTTPError, URLError):
pass
else:
if contents is not None:
return contents
# Fallback to searching within the source code.
code = substitutions.get('language', config.mailman.default_language)
# Find the template, mutating any missing template exception.
missing = object()
default_uri = ALL_TEMPLATES.get(name, missing)
if default_uri is None:
return ''
elif default_uri is missing:
raise URLError('No such file')
path, fp = find(default_uri, mlist, code)
try:
return fp.read()
finally:
fp.close()
def get(url, **kws):
parsed = urlparse(url)
if parsed.scheme in ('http', 'https'):
response = requests.get(url, **kws)
response.raise_for_status()
return response.text
if parsed.scheme == 'file':
mode = kws.pop('mode', 'r')
arguments = dict(mode=mode)
if 'encoding' in kws or 'b' not in mode:
arguments['encoding'] = kws.pop('encoding', 'utf-8')
if len(kws) > 0:
raise ValueError('Unexpected arguments: {}'.format(
COMMASPACE.join(sorted(kws))))
with open(parsed.path, **arguments) as fp:
return fp.read()
if parsed.scheme == 'mailman':
mlist = code = None
if len(kws) > 0:
raise ValueError('Unexpected arguments: {}'.format(
COMMASPACE.join(sorted(kws))))
# The path can contain one, two, or three components. Since no empty
# path components are legal, filter them out.
parts = [p for p in parsed.path.split('/') if p]
if len(parts) == 0:
raise URLError('No template specified')
elif len(parts) == 1:
template = parts[0]
elif len(parts) == 2:
part0, template = parts
# Is part0 a language code or a mailing list? This is rather
# tricky because if it's a mailing list, it could be a list-id and
# that will contain dots, as could the language code.
language = getUtility(ILanguageManager).get(part0)
if language is None:
list_manager = getUtility(IListManager)
# part0 must be a fqdn-listname or list-id.
mlist = (list_manager.get(part0) if '@' in part0 else
list_manager.get_by_list_id(part0))
if mlist is None:
raise URLError('Bad language or list name')
else:
code = language.code
elif len(parts) == 3:
part0, code, template = parts
# part0 could be an fqdn-listname or a list-id.
mlist = (getUtility(IListManager).get(part0) if '@' in part0 else
getUtility(IListManager).get_by_list_id(part0))
if mlist is None:
raise URLError('Missing list')
language = getUtility(ILanguageManager).get(code)
if language is None:
raise URLError('No such language')
code = language.code
else:
raise URLError('No such file')
# Find the template, mutating any missing template exception.
try:
path, fp = find(template, mlist, code)
except TemplateNotFoundError:
raise URLError('No such file')
try:
return fp.read()
finally:
fp.close()
raise URLError(url)
def _get_slice_data(slc: Slice, delivery_type: EmailDeliveryType,
session: Session) -> ReportContent:
slice_url = _get_url_path("Superset.explore_json",
csv="true",
form_data=json.dumps({"slice_id": slc.id}))
# URL to include in the email
slice_url_user_friendly = _get_url_path("Superset.slice",
slice_id=slc.id,
user_friendly=True)
# Login on behalf of the "reports" user in order to get cookies to deal with auth
auth_cookies = machine_auth_provider_factory.instance.get_auth_cookies(
get_reports_user(session))
# Build something like "session=cool_sess.val;other-cookie=awesome_other_cookie"
cookie_str = ";".join(
[f"{key}={val}" for key, val in auth_cookies.items()])
opener = urllib.request.build_opener()
opener.addheaders.append(("Cookie", cookie_str))
response = opener.open(slice_url)
if response.getcode() != 200:
raise URLError(response.getcode())
# TODO: Move to the csv module
content = response.read()
rows = [r.split(b",") for r in content.splitlines()]
if delivery_type == EmailDeliveryType.inline:
data = None
# Parse the csv file and generate HTML
columns = rows.pop(0)
with app.app_context():
body = render_template(
"superset/reports/slice_data.html",
columns=columns,
rows=rows,
name=slc.slice_name,
link=slice_url_user_friendly,
)
elif delivery_type == EmailDeliveryType.attachment:
data = {__("%(name)s.csv", name=slc.slice_name): content}
body = __(
'<b><a href="%(url)s">Explore in Superset</a></b><p></p>',
name=slc.slice_name,
url=slice_url_user_friendly,
)
# how to: https://api.slack.com/reference/surfaces/formatting
slack_message = __(
"""
*%(slice_name)s*\n
<%(slice_url_user_friendly)s|Explore in Superset>
""",
slice_name=slc.slice_name,
slice_url_user_friendly=slice_url_user_friendly,
)
return ReportContent(body, data, None, slack_message, content)
def test_facebook_failed_request(self):
config = {'side_effect': URLError(reason="Testing error handling")}
with patch.object(urllib.request, 'urlopen', **config):
self.assertRaises(EmbedNotFoundException,
FacebookOEmbedFinder().find_embed,
"https://fb.watch/ABC123eew/")
def mocked_get_addr_info_failed(self, request):
raise URLError('Getaddrinfo failed')
def mocked_connection_time_out(self, request):
raise URLError('Connection timed out')
def mock_urlopen(*args, **kwargs):
raise URLError("http://foo")
},
"files": [
{
"path": "src/main.c",
"status": "modified",
"additions": 1,
"deletions": 1
}
]
}
}''')), # valid response
('CVE-1970-1000', 200, b'', None, io.BytesIO(b'')), # empty response
('CVE-1970-1000', 404, b'vcs not found',
HTTPError('/', 404, 'vcs not found', {},
io.StringIO('vcs not found')), None), # not found
('CVE-1970-1000', 400, b'something wrong', URLError('something wrong'),
None), # url error
('CVE-1970-1500', 404, b'not found', None, 'None'), # no commit
('CVE-1970-9000', 404, b'not found', None, None), # cve not found
('9999', 404, b'not found', None, None), # id not found
('Invalid ID', 404, b'not found', None, None), # invalid id
]
@pytest.mark.integration
@pytest.mark.parametrize("input_id,expected_status,expected_content",
VULN_VIEW_VARIANTS)
def test_view_vulnerability(client, input_id, expected_status,
expected_content):
resp = client.get('/' + input_id)
assert resp.status_code == expected_status
# Test `rhessi.get_base_url()`
@mock.patch('sunpy.net.dataretriever.sources.rhessi.urlopen',
return_value=None)
def test_get_base_url(mock_urlopen):
"""
Success case, can successfully 'ping' first data_server
"""
assert rhessi.get_base_url() == rhessi.data_servers[0]
@mock.patch('sunpy.net.dataretriever.sources.rhessi.urlopen',
side_effect=URLError(''))
def test_get_base_url_on_urlerror(mock_urlopen):
"""
If all tested URLs raise `URLError`, then raise an `IOError`
"""
with pytest.raises(IOError):
rhessi.get_base_url()
@mock.patch('sunpy.net.dataretriever.sources.rhessi.urlopen',
side_effect=socket.timeout)
def test_get_base_url_on_timeout(mock_urlopen):
"""
If all tested data servers timeout, then raise an `IOError`
"""
with pytest.raises(IOError):
def test_oembed_invalid_request(self):
config = {'side_effect': URLError('foo')}
with patch.object(urllib.request, 'urlopen', **config):
self.assertRaises(EmbedNotFoundException,
OEmbedFinder().find_embed,
"http://www.youtube.com/watch/")
def dummy_urlretrieve(url, local_path):
local_path.touch()
raise URLError("Test")
def test_instagram_failed_request(self):
config = {'side_effect': URLError(reason="Testing error handling")}
with patch.object(urllib.request, 'urlopen', **config):
self.assertRaises(EmbedNotFoundException,
InstagramOEmbedFinder().find_embed,
"https://instagr.am/p/CHeRxmnDSYe/")
class TestImageLoader(unittest.TestCase):
def setUp(self) -> None:
self.image_loader = ImageLoader()
self.im_paths = [image_name_to_path(f) for f in TEST_IMAGES]
self.im_url = "http://file.biolab.si/images/bone-healing/D14/D14-" \
"0401-11-L1-inj-1-0016-m1.jpg"
def test_load_images(self) -> None:
image = self.image_loader.load_image_or_none(self.im_paths[0])
self.assertTrue(isinstance(image, Image))
image = self.image_loader.load_image_or_none(self.im_paths[0],
target_size=(255, 255))
self.assertTrue(isinstance(image, Image))
self.assertTupleEqual((255, 255), image.size)
def test_load_images_url(self) -> None:
"""
Handle loading images from http, https type urls
"""
image = self.image_loader.load_image_or_none(self.im_url)
self.assertTrue(isinstance(image, Image))
image = self.image_loader.load_image_or_none(self.im_paths[0],
target_size=(255, 255))
self.assertTrue(isinstance(image, Image))
self.assertTupleEqual((255, 255), image.size)
# invalid urls could be handled
image = self.image_loader.load_image_or_none(self.im_url + "a")
self.assertIsNone(image)
@patch("requests.sessions.Session.get", side_effect=RequestException)
def test_load_images_url_request_exception(self, _) -> None:
"""
Handle loading images from http, https type urls
"""
image = self.image_loader.load_image_or_none(self.im_url)
self.assertIsNone(image)
@patch("orangecontrib.imageanalytics.utils.embedder_utils.urlopen",
return_value=image_name_to_path(TEST_IMAGES[0]))
def test_load_images_ftp(self, _) -> None:
"""
Handle loading images from ftp, data type urls. Since we do not have
a ftp source we just change path to local path.
"""
image = self.image_loader.load_image_or_none("ftp://abcd")
self.assertTrue(isinstance(image, Image))
image = self.image_loader.load_image_or_none(self.im_paths[0],
target_size=(255, 255))
self.assertTrue(isinstance(image, Image))
self.assertTupleEqual((255, 255), image.size)
@patch("orangecontrib.imageanalytics.utils.embedder_utils.urlopen",
side_effect=URLError("wrong url"))
def test_load_images_ftp_error(self, _) -> None:
"""
Handle loading images from ftp, data type urls. Since we do not have
a ftp source we just change path to local path.
"""
image = self.image_loader.load_image_or_none("ftp://abcd")
self.assertIsNone(image)
def test_load_image_bytes(self) -> None:
for image in self.im_paths:
image_bytes = self.image_loader.load_image_bytes(image)
self.assertTrue(isinstance(image_bytes, bytes))
# one with wrong path to get none
image_bytes = self.image_loader.load_image_bytes(self.im_paths[0] +
"a")
self.assertIsNone(image_bytes)
@patch("PIL.Image.Image.convert", side_effect=ValueError())
def test_unsuccessful_convert_to_RGB(self, _) -> None:
image = self.image_loader.load_image_or_none(self.im_paths[2])
self.assertIsNone(image)
soup = BeautifulSoup(html, 'html.parser')
# Колонка 6, итого нечет
total_odd = soup.find(id='mainidofrow6idofcell3').getText().replace(',', '.')
# Колонка 6, итого чет
total_even = soup.find(id='mainidofrow12idofcell3').getText().replace(',', '.')
# Колонка 4, ИТОГО
total_4 = soup.find(id='mainidofrow18idofcell1').getText().replace(',', '.')
# Колонка 5, ИТОГО
total_5 = soup.find(id='mainidofrow18idofcell2').getText().replace(',', '.')
# Колонка 7, ИТОГО
total_7 = soup.find(id='mainidofrow18idofcell4').getText().replace(',', '.')
# Колонка 9, ИТОГО
total_9 = soup.find(id='mainidofrow18idofcell6').getText().replace(',', '.')
except URLError as e:
print(f"URLErorr e={e}")
raise URLError(e)
# todo: продумать логирование ошибки, мониторинг и реакцию на нее (если страницу не удалось загрузить)
# запись данных в excel
try:
wb = load_workbook(os.path.join(CURRENT_DIR, XLSX_TEMPLATE))
sheet = wb['trains']
row = 1
# находим первую незаполненную строку
while row < XLSX_MAX_ROW:
value = sheet.cell(row=row, column=1).value
if not value:
break
row += 1
if row < XLSX_MAX_ROW:
sheet.cell(row=row, column=1).value = NOW
def test_client_had_exception(self):
self.assertSendsEmail(
{'body': self.test_client_had_exception.__name__},
success=False,
exception=URLError('sendgrid error'))
