def install_mpt(install_path, url=DEFAULT_MPT_URL):
"""
Install MyPyTutor to the given directory.
Args:
install_path (str): The directory to install MyPyTutor in.
url (str, optional): The URL of the MyPyTutor file to use.
"""
# create our install path if it doesn't already exist
if not os.path.exists(install_path):
os.makedirs(install_path)
print('Installing MyPyTutor...', end='', flush=True)
# grab the latest zip file
# we use an explicit filename here because we don't yet have access
# to the tutorlib module for abstracting away temporary file creation
try:
urlobj = URLopener()
filename, _ = urlobj.retrieve(url, 'MyPyTutor.zip')
except Exception:
print('failed')
sys.exit(1)
# extract the file
with ZipFile(filename) as zf:
zf.extractall(install_path)
print('done')
def read_file_content(self, file_url=None):
"""Return name of temp file in which remote file is saved."""
if not file_url:
file_url = self.url
pywikibot.warning("file_url is not given. "
"Set to self.url by default.")
pywikibot.output(u'Reading file %s' % file_url)
resume = False
rlen = 0
_contents = None
dt = 15
uo = URLopener()
retrieved = False
while not retrieved:
if resume:
pywikibot.output(u"Resume download...")
uo.addheader('Range', 'bytes=%s-' % rlen)
infile = uo.open(file_url)
if 'text/html' in infile.info().getheader('Content-Type'):
pywikibot.output(u"Couldn't download the image: "
"the requested URL was not found on server.")
return
content_len = infile.info().getheader('Content-Length')
accept_ranges = infile.info().getheader('Accept-Ranges') == 'bytes'
if resume:
_contents += infile.read()
else:
_contents = infile.read()
infile.close()
retrieved = True
if content_len:
rlen = len(_contents)
content_len = int(content_len)
if rlen < content_len:
retrieved = False
pywikibot.output(
u"Connection closed at byte %s (%s left)"
% (rlen, content_len))
if accept_ranges and rlen > 0:
resume = True
pywikibot.output(u"Sleeping for %d seconds..." % dt)
time.sleep(dt)
if dt <= 60:
dt += 15
elif dt < 360:
dt += 60
else:
pywikibot.log(
u"WARNING: length check of retrieved data not possible.")
handle, tempname = tempfile.mkstemp()
with os.fdopen(handle, "wb") as t:
t.write(_contents)
return tempname
def get_imagelinks(url):
"""Given a URL, get all images linked to by the page at that URL."""
# Check if BeautifulSoup is imported.
if isinstance(BeautifulSoup, ImportError):
raise BeautifulSoup
links = []
uo = URLopener()
with uo.open(url) as f:
soup = BeautifulSoup(f.read())
if not shown:
tagname = "a"
elif shown == "just":
tagname = "img"
else:
tagname = ["a", "img"]
for tag in soup.findAll(tagname):
link = tag.get("src", tag.get("href", None))
if link:
ext = os.path.splitext(link)[1].lower().strip('.')
if ext in fileformats:
links.append(urllib.basejoin(url, link))
return links
def readTLEfile(source):
''' Read a TLE file (unzip if necessary) '''
sourceName = source['name']
sourceUrl = source['url']
sourceFile = source['file']
if os.path.isfile(sourceFile):
print('Using saved TLE data {} ({})'.format(
sourceFile, time.ctime(os.path.getmtime(sourceFile))))
else:
print('Retrieving TLE data from {}'.format(sourceUrl))
file = URLopener()
try:
file.retrieve(sourceUrl, sourceFile)
except:
print("Error: Failed to get TLE data")
return None
else:
print('{} updated'.format(sourceFile))
if sourceFile.lower().endswith('.zip'):
print('Unzipping {}...'.format(sourceFile))
zip = zipfile.ZipFile(sourceFile)
zip.extractall('.')
sourceFile = zip.namelist()[0]
print('Extracted {}'.format(zip.namelist()))
tempContent = []
with open(sourceFile) as f:
for aline in f:
tempContent.append(aline.replace('\n', ''))
print(len(tempContent) // 3, 'TLEs loaded from {}'.format(sourceFile))
return tempContent
def install_mpt(install_path, url=DEFAULT_MPT_URL):
"""
Install MyPyTutor to the given directory.
Args:
install_path (str): The directory to install MyPyTutor in.
url (str, optional): The URL of the MyPyTutor file to use.
"""
# create our install path if it doesn't already exist
if not os.path.exists(install_path):
os.makedirs(install_path)
print('Installing MyPyTutor...', end='', flush=True)
# grab the latest zip file
# we use an explicit filename here because we don't yet have access
# to the tutorlib module for abstracting away temporary file creation
try:
urlobj = URLopener()
filename, _ = urlobj.retrieve(url, 'MyPyTutor.zip')
except Exception:
print('failed')
sys.exit(1)
# extract the file
with ZipFile(filename) as zf:
zf.extractall(install_path)
print('done')
def get_imagelinks(url):
"""Given a URL, get all images linked to by the page at that URL."""
# Check if BeautifulSoup is imported.
if isinstance(BeautifulSoup, ImportError):
raise BeautifulSoup
links = []
uo = URLopener()
with uo.open(url) as f:
soup = BeautifulSoup(f.read())
if not shown:
tagname = "a"
elif shown == "just":
tagname = "img"
else:
tagname = ["a", "img"]
for tag in soup.findAll(tagname):
link = tag.get("src", tag.get("href", None))
if link:
ext = os.path.splitext(link)[1].lower().strip('.')
if ext in fileformats:
links.append(urllib.basejoin(url, link))
return links
def read_file_content(self, file_url=None):
"""Return name of temp file in which remote file is saved."""
if not file_url:
file_url = self.url
pywikibot.warning("file_url is not given. "
"Set to self.url by default.")
pywikibot.output(u'Reading file %s' % file_url)
resume = False
rlen = 0
_contents = None
dt = 15
uo = URLopener()
retrieved = False
while not retrieved:
if resume:
pywikibot.output(u"Resume download...")
uo.addheader('Range', 'bytes=%s-' % rlen)
infile = uo.open(file_url)
if 'text/html' in infile.info().getheader('Content-Type'):
pywikibot.output(u"Couldn't download the image: "
"the requested URL was not found on server.")
return
content_len = infile.info().getheader('Content-Length')
accept_ranges = infile.info().getheader('Accept-Ranges') == 'bytes'
if resume:
_contents += infile.read()
else:
_contents = infile.read()
infile.close()
retrieved = True
if content_len:
rlen = len(_contents)
content_len = int(content_len)
if rlen < content_len:
retrieved = False
pywikibot.output(
u"Connection closed at byte %s (%s left)" %
(rlen, content_len))
if accept_ranges and rlen > 0:
resume = True
pywikibot.output(u"Sleeping for %d seconds..." % dt)
time.sleep(dt)
if dt <= 60:
dt += 15
elif dt < 360:
dt += 60
else:
pywikibot.log(
u"WARNING: length check of retrieved data not possible.")
handle, tempname = tempfile.mkstemp()
with os.fdopen(handle, "wb") as t:
t.write(_contents)
return tempname
def downloadZipFromLink(dowLink, albumName):
zipName = albumName.replace(" ", "") + ".zip"
tempDir = tempfile.gettempdir();
zipPath = os.path.join(tempDir, zipName)
if (os.path.exists(zipPath)):
print("Deleting zip which already exists at location '{}'".format(zipPath));
os.remove(zipPath);
print("Downloading '{}' from '{}' to file '{}'".format(albumName, dowLink, zipPath))
urlopener = URLopener();
def reporthook(blocknum, blocksize, totalsize):
readsofar = blocknum * blocksize
if totalsize > 0:
percent = readsofar * 1e2 / totalsize
s = "\r%5.1f%% %*d / %d" % (
percent, len(str(totalsize)), readsofar, totalsize)
sys.stderr.write(s)
if readsofar >= totalsize: # near the end
sys.stderr.write("\n")
else: # total size is unknown
sys.stderr.write("read %d\n" % (readsofar,))
try:
urlopener.retrieve(dowLink, zipPath, reporthook)
except:
if (os.path.exists(zipPath)):
print("\nDeleting archive file '{}'".format(zipPath));
os.remove(zipPath);
raise;
print("Download of '{}' has completed.".format(albumName))
return zipPath;
def read(self, uri=None, resources=None, index_only=False):
"""Read sitemap from a URI including handling sitemapindexes.
If index_only is True then individual sitemaps references in a sitemapindex
will not be read. This will result in no resources being returned and is
useful only to read the metadata and links listed in the sitemapindex.
Includes the subtlety that if the input URI is a local file and is a
sitemapindex which contains URIs for the individual sitemaps, then these
are mapped to the filesystem also.
"""
try:
fh = URLopener().open(uri)
self.num_files += 1
except IOError as e:
raise IOError(
"Failed to load sitemap/sitemapindex from %s (%s)" %
(uri, str(e)))
# Get the Content-Length if we can (works fine for local files)
try:
self.content_length = int(fh.info()['Content-Length'])
self.bytes_read += self.content_length
self.logger.debug(
"Read %d bytes from %s" %
(self.content_length, uri))
except KeyError:
# If we don't get a length then c'est la vie
self.logger.debug("Read ????? bytes from %s" % (uri))
pass
self.logger.info("Read sitemap/sitemapindex from %s" % (uri))
s = self.new_sitemap()
s.parse_xml(fh=fh, resources=self, capability=self.capability_name)
# what did we read? sitemap or sitemapindex?
if (s.parsed_index):
# sitemapindex
if (not self.allow_multifile):
raise ListBaseIndexError(
"Got sitemapindex from %s but support for sitemapindex disabled" %
(uri))
self.logger.info(
"Parsed as sitemapindex, %d sitemaps" %
(len(
self.resources)))
sitemapindex_is_file = self.is_file_uri(uri)
if (index_only):
# don't read the component sitemaps
self.sitemapindex = True
return
# now loop over all entries to read each sitemap and add to
# resources
sitemaps = self.resources
self.resources = self.resources_class()
self.logger.info("Now reading %d sitemaps" % len(sitemaps.uris()))
for sitemap_uri in sorted(sitemaps.uris()):
self.read_component_sitemap(
uri, sitemap_uri, s, sitemapindex_is_file)
else:
# sitemap
self.logger.info("Parsed as sitemap, %d resources" %
(len(self.resources)))
def download_file(self):
"""
this function will visit a url for a specific location, enter the date
and save the file to a specdified directory
# http://penteli.meteo.gr/meteosearch/data/aghiosnikolaos/2009-11.txt
"""
for station in self.stations['stations'][:]:
try:
# os.mkdir('./data/' + station)
os.mkdir(os.path.join(os.getcwd(), data_folder) + '/' + station) # messy!!!
except:
# add logging and fix exceptions too broad
print('directory: {0} all ready exists!!!'.format(station))
pass
testfile = URLopener()
os.chdir(data_folder + '/' + station)
for i, date in enumerate(self.dates_to_download):
name_to_save_file = os.getcwd() + '/' + station + '-' + date + '.txt'
print(os.getcwd())
try:
# this is the complete url to visit and download its contents
url = url_seed + station + '/' + date + '.txt'
testfile.retrieve(url, name_to_save_file)
except:
pass
os.chdir(os.pardir)
os.chdir(os.pardir)
def _verify_and_download(self):
"""check if file is where it should and download if not"""
if path.isfile(self._path):
return
# File does not exist, so we have to download it.
epic_id = int(self.epic_id)
d1 = epic_id - epic_id % 100000
d2 = epic_id % 100000 - epic_id % 1000
url_template = 'https://archive.stsci.edu/missions/k2/target_pixel_files/c{0:d}/{1:d}/{2:05d}/{3}'
url_to_load = url_template.format(self.campaign, d1, d2,
self.file_name)
fmt = "Downloading {:} ..... "
print(fmt.format(self.file_name), end='', file=sys.stderr, flush=True)
url_retriever = URLopener()
try:
url_retriever.retrieve(url_to_load, self._path)
except exceptions:
print("", file=sys.stderr, flush=True)
raise IOError(
"\n\nFailed to download file {:}\n\n".format(url_to_load))
if not path.isfile(self._path):
print("", file=sys.stderr, flush=True)
raise IOError('Download of\n' + url_to_load + '\nto\n' +
self._path + 'somehow failed')
print(" done", file=sys.stderr, flush=True)
def download_data(admin_level, plz_regex_string, filename):
bbox = '48.07303233901773,11.348190307617188,48.25028349849019,11.73614501953125'
query = 'rel(' +bbox + ')[boundary=administrative][admin_level={}]; out geom;'.format(admin_level) + \
'rel(' +bbox + ')[boundary=postal_code][postal_code~"{}"]; out geom;'.format(plz_regex_string)
file = URLopener()
file.retrieve('http://overpass-api.de/api/interpreter?data=' + quote_plus(query), filename)
def readTLEfile(source):
''' Read a TLE file (unzip if necessary) '''
sourceName = source['name']
sourceUrl = source['url']
sourceFile = source['file']
if os.path.isfile(sourceFile):
print('Using saved TLE data {} ({})'.format(sourceFile,
time.ctime(os.path.getmtime(sourceFile))))
else:
print('Retrieving TLE data from {}'.format(sourceUrl))
file = URLopener()
try:
file.retrieve(sourceUrl, sourceFile)
except:
print("Error: Failed to get TLE data")
return None
else:
print('{} updated'.format(sourceFile))
if sourceFile.lower().endswith('.zip'):
print('Unzipping {}...'.format(sourceFile))
zip = zipfile.ZipFile(sourceFile)
zip.extractall('.')
sourceFile = zip.namelist()[0]
print('Extracted {}'.format(zip.namelist()))
tempContent = []
with open(sourceFile) as f:
for aline in f:
tempContent.append(aline.replace('\n', ''))
print(len(tempContent) // 3,
'TLEs loaded from {}'.format(sourceFile))
return tempContent
def read_component_sitemap(self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file):
"""Read a component sitemap of a Resource List with index.
Each component must be a sitemap with the
"""
if (sitemapindex_is_file):
if (not self.is_file_uri(sitemap_uri)):
# Attempt to map URI to local file
remote_uri = sitemap_uri
sitemap_uri = self.mapper.src_to_dst(remote_uri)
self.logger.info("Mapped %s to local file %s" % (remote_uri, sitemap_uri))
else:
# The individual sitemaps should be at a URL (scheme/server/path)
# that the sitemapindex URL can speak authoritatively about
if (self.check_url_authority and
not UrlAuthority(sitemapindex_uri).has_authority_over(sitemap_uri)):
raise ListBaseIndexError("The sitemapindex (%s) refers to sitemap at a location it does not have authority over (%s)" % (sitemapindex_uri,sitemap_uri))
try:
fh = URLopener().open(sitemap_uri)
self.num_files += 1
except IOError as e:
raise ListBaseIndexError("Failed to load sitemap from %s listed in sitemap index %s (%s)" % (sitemap_uri,sitemapindex_uri,str(e)))
# Get the Content-Length if we can (works fine for local files)
try:
self.content_length = int(fh.info()['Content-Length'])
self.bytes_read += self.content_length
except KeyError:
# If we don't get a length then c'est la vie
pass
self.logger.info( "Reading sitemap from %s (%d bytes)" % (sitemap_uri,self.content_length) )
component = sitemap.parse_xml( fh=fh, sitemapindex=False )
# Copy resources into self, check any metadata
for r in component:
self.resources.add(r)
def call_api(url, wait=1):
time.sleep(wait)
req = URLopener()
req.addheader('Authorization', 'token ' + TOKEN)
fp = req.open(url)
data = json.load(fp)
fp.close()
return data
def read(self, uri=None, resources=None, index_only=False):
"""Read sitemap from a URI including handling sitemapindexes.
If index_only is True then individual sitemaps references in a sitemapindex
will not be read. This will result in no resources being returned and is
useful only to read the metadata and links listed in the sitemapindex.
Includes the subtlety that if the input URI is a local file and is a
sitemapindex which contains URIs for the individual sitemaps, then these
are mapped to the filesystem also.
"""
try:
fh = URLopener().open(uri)
self.num_files += 1
except IOError as e:
raise IOError("Failed to load sitemap/sitemapindex from %s (%s)" %
(uri, str(e)))
# Get the Content-Length if we can (works fine for local files)
try:
self.content_length = int(fh.info()['Content-Length'])
self.bytes_read += self.content_length
self.logger.debug("Read %d bytes from %s" %
(self.content_length, uri))
except KeyError:
# If we don't get a length then c'est la vie
self.logger.debug("Read ????? bytes from %s" % (uri))
pass
self.logger.info("Read sitemap/sitemapindex from %s" % (uri))
s = self.new_sitemap()
s.parse_xml(fh=fh, resources=self, capability=self.capability_name)
# what did we read? sitemap or sitemapindex?
if (s.parsed_index):
# sitemapindex
if (not self.allow_multifile):
raise ListBaseIndexError(
"Got sitemapindex from %s but support for sitemapindex disabled"
% (uri))
self.logger.info("Parsed as sitemapindex, %d sitemaps" %
(len(self.resources)))
sitemapindex_is_file = self.is_file_uri(uri)
if (index_only):
# don't read the component sitemaps
self.sitemapindex = True
return
# now loop over all entries to read each sitemap and add to
# resources
sitemaps = self.resources
self.resources = self.resources_class()
self.logger.info("Now reading %d sitemaps" % len(sitemaps.uris()))
for sitemap_uri in sorted(sitemaps.uris()):
self.read_component_sitemap(uri, sitemap_uri, s,
sitemapindex_is_file)
else:
# sitemap
self.logger.info("Parsed as sitemap, %d resources" %
(len(self.resources)))
def tryToReadFile(filePath, urlData):
contents = ""
try:
fileHandle = URLopener().open(filePath)
contents = fileHandle.read()
except IOError:
contents = "mitmProxy - Resource Override: Could not open " + filePath + \
" Came from rule: " + urlData[0] + " , " + urlData[1]
return contents
class HttpFetcherBasic(HttpFetcher):
def __init__(self, url):
super().__init__(url)
self.urlop = URLopener()
for hdr, val in (tuple(x.split("=", 1)) if "=" in x else (x, "")
for x in url.fragment.split("&") if x):
self.urlop.addheader(hdr, val)
def open(self, url):
return self.urlop.open(url)
def download(self, entity_id: int, destination: str = None, sort: List[Sort] = None) -> str:
"""
Download sequences from a single entity.
"""
sort = [Sort('id', 'asc')] if sort is None else sort
sort = list(sort_item.to_json() for sort_item in sort) if sort else []
body = {'filter': [], 'selection': [], 'sort': sort}
file_path = Sequences.get_filepath_for_entity_id(entity_id)
url = '{}/entities/{}/_extract'.format(self.url, entity_id)
print('Downloading shards from "{}" to "{}".'.format(url, file_path))
paths = []
with self.session.post(url, stream=True, timeout=10 * 60, json=body) as response:
try:
links = response.json()
print('links', links)
if 'statusCode' in links and links['statusCode'] != 200:
raise Exception(links['message'])
elif len(links) == 0:
raise Exception(
'Sequences:download - Error; no download links for {}. Does the table exist?'.format(entity_id))
index = 0
for link in links:
testfile = URLopener()
path = '{}-{}.gz'.format(file_path, index)
paths.append(path)
testfile.retrieve(link, path)
index = index + 1
except Exception as e:
print('Sequences:download - error:', e)
raise e
sorted_paths = self.get_sorted_file_shard_list(entity_id, paths, [])
print(f'Unzipping: entity_id={entity_id} to destination={destination}')
skip_first = False
with open(destination, 'wb+') as target_file:
for file_shard in sorted_paths:
with gzip.open(file_shard, 'rb') as g_zip_file:
first_line = True
for line in g_zip_file:
# We skip the first line of every file, except for the very first.
if not (first_line and skip_first):
line = Sequences.sanitize(line.decode("utf-8"))
target_file.write(line.encode("utf-8"))
first_line = False
# We skip the first line of every file, except for the very first.
skip_first = True
return destination
def save_downloaded_file(context):
"""
Saves POEditor terms to a file in output dir
:param context: behave context
:return: N/A
"""
file_path = get_poeditor_file_path(context)
saved_file = URLopener()
saved_file.retrieve(context.poeditor_download_url, file_path)
context.logger.info('POEditor terms have been saved in "%s" file' %
file_path)
def run(self):
# bpo-35907, CVE-2019-9948: urllib must reject local_file:// scheme
class DummyURLopener(URLopener):
def open_local_file(self, url):
return url
warnings.simplefilter('ignore', DeprecationWarning)
self.check_func("urlopen", urllib_request.urlopen)
self.check_func("URLopener().open", URLopener().open)
self.check_func("URLopener().retrieve", URLopener().retrieve)
self.check_func("DummyURLopener().open", DummyURLopener().open)
self.check_func("DummyURLopener().retrieve", DummyURLopener().retrieve)
self.exit_fixed()
def download_text_file(url, file_name):
opener = URLopener()
file_name = file_name.split("/")[-1]
file_name = file_name.replace("%20", " ")
if _is_absolute_link(file_name):
url = file_name
if not url.startswith("http://"):
url = "http://" + url
out_name = file_name.split("/")[-1]
else:
url = "{}{}".format(url, file_name)
out_name = file_name
opener.retrieve(url, file_name)
return out_name
def download_data():
"""This function downloads the data, extract them and remove the archive."""
if not os.path.exists(DATA_HOME):
print("Data are missing. Downloading them now...", end="", flush=True)
datafile = URLopener()
datafile.retrieve(DOWNLOAD_URL, ARCHIVE_FNAME)
print("Ok.")
print("Extracting now...", end="", flush=True)
tf = tarfile.open(ARCHIVE_FNAME)
tf.extractall()
print("Ok.")
print("Removing the archive...", end="", flush=True)
os.remove(ARCHIVE_FNAME)
print("Ok.")
def downloadFile(linkStore):
for imgUrl in linkStore:
try:
#removing double slash from the start of url
imgUrl = urlEdit(imgUrl[2:])
fileName = imgUrl.split("/")[-1]
imgUrl = 'https://' + imgUrl
print('Downloading file: ' + fileName + '\tURL: ' + imgUrl + '\n')
image = URLopener()
image.retrieve(imgUrl, fileName)
# above line may create error due to 403 forbidden response
except:
print("Error occured while downloading file: " + imgUrl + '\n')
continue
def _download_file(url, destination):
logger.info('Downloading %s to %s...', url, destination)
response = _open_url(url)
if not response.code == 200:
raise WagonError("Failed to download file. Request to {0} "
"failed with HTTP Error: {1}".format(
url, response.code))
final_url = response.geturl()
if final_url != url and is_verbose():
logger.debug('Redirected to %s', final_url)
f = URLopener()
f.retrieve(final_url, destination)
def main():
username = input("username: "******"password: "******"http://www.loxa.edu.tw/index.php")
with urlopen(r) as response:
phpsessid = response.getheader("set-cookie").split("; ")[0].split("=")[1]
cookie = "PHPSESSID={0}; Cookie_Allow=1".format(phpsessid)
data = {"loginname": username, "loginpswd": password}
r = Request(
"http://www.loxa.edu.tw/check.php",
data=urlencode(data).encode("utf8"),
headers={"cookie": cookie},
method="POST",
)
try:
response = urlopen(r)
except HTTPError:
sys.exit("Invalid username or password.")
r = Request("http://www.loxa.edu.tw/index.php?login=1&show_msg=Y", headers={"cookie": cookie})
response = urlopen(r)
r = Request("http://www.loxa.edu.tw/jewelbox/foldertree.php", headers={"cookie": cookie})
with urlopen(r) as response:
html = response.read().decode("big5")
folder_tree_pattern = re.compile('insFld\(.+?, gFld\(".+?", "file_list.php\?dir_id=(\d+?)", "\w"\)\);')
file_url_pattern = re.compile('<td colspan=3 nowrap>\s+?<a href="(http.+?)"')
for i in folder_tree_pattern.finditer(html):
dir_id = i.group(1)
r = Request(
"http://www.loxa.edu.tw/jewelbox/file_list.php?dir_id={0}".format(dir_id), headers={"cookie": cookie}
)
with urlopen(r) as response:
html = response.read().decode("big5")
for i in file_url_pattern.finditer(html):
url = i.group(1)
url_data = urlparse(url)
file_path = url_data.path.lstrip("/")
dir_name, base_name = os.path.split(file_path)
if not os.path.exists(dir_name):
os.makedirs(dir_name)
url_opener = URLopener()
url_opener.addheader("cookie", cookie)
print("Download: {0} -> {1}".format(url, file_path))
url_opener.retrieve(url, file_path)
def on_update_button_click(self):
try:
opener = URLopener()
opener.retrieve(self.REMOTE_UPDATE_URL,
"resources/parameters.json")
# Read the new settings.
self.data = read_settings()
messagebox.showinfo(
"Settings Update",
"Settings successfully updated from the server.")
except Exception as e:
logging.critical(
"Couldn't open the remote settings file: {0}".format(str(e)))
messagebox.showerror("Couldn't Update Settings",
"Couldn't open the remote settings file.")
def initUrllibInstance(self, server):
printl("", self, "S")
# we establish the connection once here
self.urllibInstance = URLopener()
# we add headers only in special cases
connectionType = self.serverConfig.connectionType.value
localAuth = self.serverConfig.localAuth.value
if connectionType == "2" or localAuth:
authHeader = self.plexInstance.get_hTokenForServer(server)
self.urllibInstance.addheader("X-Plex-Token",
authHeader["X-Plex-Token"])
printl("", self, "C")
class RemoteFile(object):
def __init__(self, url):
self.opener = URLopener()
self.url = url
self.filename = url.rstrip('/').rsplit('/', 1)[-1]
self.offset = 0
def seek(self, offset, whence=0):
assert whence == 0
self.offset = offset
def read(self, size):
start = self.offset
end = start + size - 1
assert end > start
h = 'Range', 'bytes={}-{}'.format(start, end)
stderr.write('Fetching {} {}\n'.format(self.filename, h[1]))
self.opener.addheaders.append(h)
data = self.opener.open(self.url).read()
return data
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def parse(self, uri=None, fh=None, str_data=None, **kwargs):
"""Parse a single XML document for this list.
Accepts either a uri (uri or default if parameter not specified),
or a filehandle (fh) or a string (str_data). Note that this method
does not handle the case of a sitemapindex+sitemaps.
LEGACY SUPPORT - the parameter str may be used in place of str_data
but is deprecated and will be removed in a later version.
"""
if (uri is not None):
try:
fh = URLopener().open(uri)
except IOError as e:
raise Exception(
"Failed to load sitemap/sitemapindex from %s (%s)" %
(uri, str(e)))
elif (str_data is not None):
fh = io.StringIO(str_data)
elif ('str' in kwargs):
# Legacy support for str argument, see
# https://github.com/resync/resync/pull/21
# One test for this in tests/test_list_base.py
self.logger.warn(
"Legacy parse(str=...), use parse(str_data=...) instead")
fh = io.StringIO(kwargs['str'])
if (fh is None):
raise Exception("Nothing to parse")
s = self.new_sitemap()
s.parse_xml(fh=fh,
resources=self,
capability=self.capability_name,
sitemapindex=False)
self.parsed_index = s.parsed_index
def __init__(self, url):
super(HttpDB, self).__init__()
self.baseurl = url._replace(fragment="").geturl()
self.db = urlop = URLopener()
for hdr, val in (tuple(x.split("=", 1)) if "=" in x else (x, "")
for x in url.fragment.split("&") if x):
urlop.addheader(hdr, val)
def Download_File(name):
""" Download UCAC4 file. """
url_name = prefix+name
ucac_file = URLopener()
ucac_file.retrieve(url_name, name)
inp = open(name, 'rb')
bz2_file = bz2.BZ2File(name+'.bz2', 'wb', compresslevel=1)
copyfileobj(inp, bz2_file)
inp.close()
bz2_file.close()
os.remove(name)
return 0
def pose_video(datum):
if not exists(datum['video']):
try:
URLopener().retrieve(datum["video_url"], datum["video"])
except Exception as e:
makedir(datum["pose_dir"]) # Empty directory
if not exists(datum["pose_dir"]):
gpu = get_empty_gpu()
# Create Container
container_id = Docker.create_container(DOCKER_NAME, "-it -v " + datum["video"] + ":/video.mp4")
def remove_container():
Docker.remove_container(container_id)
try:
# Start Container
Docker.start_container(container_id)
cmd = "./build/examples/openpose/openpose.bin --video /video.mp4 --model_pose BODY_25 --display 0 --render_pose 0 --write_json /out/ --hand --face --num_gpu 1 "
cmd += " --num_gpu_start " + str(gpu)
Docker.exec_container(container_id, "bash -c 'cd /openpose && " + cmd + "'")
# Copy files
Docker.cp_container_directory(container_id, datum["pose_dir"], "/out/")
except Exception as e:
remove_container()
raise e
finally:
remove_container()
return True
def stop(self):
"""Stops the server."""
self.stop_serving = True
try:
# This is to force stop the server loop
URLopener().open("http://%s:%d" % (self.host, self.port))
except IOError:
pass
log.info("Shutting down the webserver")
self.thread.join()
def stop(self) -> None:
"""Stops the server."""
self.stop_serving = True
try:
# This is to force stop the server loop
URLopener().open(f"http://{self.host}:{self.port}")
except OSError:
pass
log.info("Shutting down the webserver")
self.thread.join()
def export(self,
entity_id: int,
format: ExportFormat,
destination_folder: str = None):
entity = self.entities.get(entity_id)
entity_name = entity['name']
user = self.authentication.user
path_parts = entity['path'].split('.')
# Last path part is always the current document.
# Any before that are ancestor folders, the first being the parent.
parent_folder_id = int(path_parts[-2]) if len(path_parts) > 1 else None
job_id = self.jobs.create(owner_id=user['orgs'][0]['id'],
shareable_id=entity['ownerId'],
job_type=JobType.ExportJob,
name='Export from python client',
input_entity_ids=[entity_id],
params={
"filter": [],
"format": format,
"fileName": entity_name,
"selection": [],
"targetFolderId": parent_folder_id,
})
# Wait for the file to be converted to Genbank.
job = self.jobs.poll_job(job_id)
links = job['outputLinks']
outputs = []
for link in links:
testfile = URLopener()
destination = os.path.join(destination_folder, entity_name)
testfile.retrieve(link['url'], destination)
outputs.append(destination)
return outputs
def get_imagelinks(url):
"""Given a URL, get all images linked to by the page at that URL."""
links = []
uo = URLopener()
file = uo.open(url)
soup = BeautifulSoup.BeautifulSoup(file.read())
file.close()
if not shown:
tagname = "a"
elif shown == "just":
tagname = "img"
else:
tagname = ["a", "img"]
for tag in soup.findAll(tagname):
link = tag.get("src", tag.get("href", None))
if link:
ext = os.path.splitext(link)[1].lower().strip('.')
if ext in fileformats:
links.append(urllib.basejoin(url, link))
return links
def get_imagelinks(url):
"""Given a URL, get all images linked to by the page at that URL."""
links = []
uo = URLopener()
file = uo.open(url)
soup = BeautifulSoup.BeautifulSoup(file.read())
file.close()
if not shown:
tagname = "a"
elif shown == "just":
tagname = "img"
else:
tagname = ["a", "img"]
for tag in soup.findAll(tagname):
link = tag.get("src", tag.get("href", None))
if link:
ext = os.path.splitext(link)[1].lower().strip('.')
if ext in fileformats:
links.append(urllib.basejoin(url, link))
return links
def online_install():
#Set location for file download by changing working directory
#Variable that stores the file name of the ninite file, the temp folder path, and the current directory
dl = 'ninite.exe'
dl_path = "c:\\Install_Wizard_Temp"
currentDir = os.getcwd()
##This should allow the download location to be changed so that the program can be run off locked flash drive
#Test to see if directory exists for program already, if not, create one
if not os.path.exists(dl_path):
os.makedirs(dl_path)
#Change working directory to one on customers computer
os.chdir(dl_path)
#Check if there is a previous ninite installer
if os.path.isfile(dl):
os.remove(dl)
print('file removed')
#Create url
url = urlCreate()
#Create object to open url
ninite = URLopener()
#Download file from url and save as installer.exe
try:
ninite.retrieve(url, dl)
except: #Error in retrieving website
text1.set('Ninite website could\nnot be accessed')
#Run the file
try:
check_call(dl, shell=True)
except: #Error in running file
text1.set('Error running ninite file')
#Test to see if dl file exists, if so, delete
if os.path.isfile(dl):
os.remove(dl)
#Change directory back to original working directory
os.chdir(currentDir)
#Check if directory that was created earlier still exists, if so remove it
if os.path.exists(dl_path):
rmtree(dl_path)
def downloadAsset(uri, dirname):
tUrl = uri
o = urlparse(tUrl)
contentType = ""
# targetDir = os.path.join(CURRENT_DIRECTORY, dirname, '/'.join(o.path.split('/')[1:-1]))
targetDir = CURRENT_DIRECTORY + '/' + dirname + '/' + '/'.join(
o.path.split('/')[1:-1])
# javascript, fragment의 경우 다운로드 불필요
if o.scheme == "javascript" or (o.netloc == '' and o.path == ''):
return
if o.scheme == "":
if uri.startswith("//"):
tUrl = f"https:{uri}"
else:
tUrl = f"https://{uri}"
try:
contentType = getContentType(tUrl)
except Exception:
try:
if uri.startswith('//'):
tUrl = f"http:{uri}"
else:
tUrl = f"http://{uri}"
contentType = getContentType(tUrl)
except Exception:
pass
# raise Exception("Error during connection")
else:
# text/html 무시
if contentType in mimeTypes[1:]:
if not os.path.exists(targetDir):
path = Path(targetDir)
path.mkdir(parents=True)
targetFile = targetDir + '/' + o.path.split('/')[-1]
if not os.path.exists(targetFile):
try:
urlretrieve(tUrl, targetFile)
print(f"[Retrieved] {targetFile}")
except Exception:
try:
opener = URLopener()
opener.addheader('User-Agent', 'Mozilla/5.0')
filename, headers = opener.retrieve(tUrl, targetFile)
except Exception:
try:
tUrl = tUrl.replace('www.', '')
tUrl = tUrl.replace('http:', 'https:')
filename, headers = opener.retrieve(
tUrl, targetFile)
except Exception as e:
print(str(e))
raise Exception
else:
pass
def open_url(url, **kwds):
"""Opens a url or file and returns an appropriate key-value reader."""
reader_cls = fileformat(url)
parsed_url = urlparse(url, 'file')
if parsed_url.scheme == 'file':
f = open(parsed_url.path, 'rb')
else:
if parsed_url.scheme == 'hdfs':
server, username, path = hdfs.urlsplit(url)
url = hdfs.datanode_url(server, username, path)
if reader_cls is ZipReader and sys.version_info < (3, 2):
# In Python <3.2, the gzip module is broken because it depends on
# the underlying file being seekable (not true for url objects).
opener = URLopener()
filename, _ = opener.retrieve(url)
f = open(filename, 'rb')
os.unlink(filename)
else:
f = urlopen(url)
return reader_cls(f, **kwds)
def download_audio_file(url, guid):
"""Download the audio file for the lecture.
Downloads the mps audio recording of the lecture. File is stored in the
tempoary folder named using the GUID
Args:
- url (str): The lecture's base url
- guid (str): The lecture's guid
"""
print("\nDownloading audio file")
URLopener().retrieve(url + "/audio.mp3",
os.path.join(DOWNLOAD_DIRECTORY, guid, "audio.mp3"))
class RemoteFile(object):
def __init__(self, url):
self.opener = URLopener()
self.url = url
self.filename = url.rstrip('/').rsplit('/', 1)[-1]
self.offset = 0
def seek(self, offset, whence=0):
assert whence == 0
self.offset = offset
def read(self, size):
start = self.offset
end = start + size - 1
assert end > start
h = 'Range', 'bytes={}-{}'.format(start, end)
stderr.write('Fetching {} {}\n'.format(self.filename, h[1]))
self.opener.addheaders.append(h)
data = self.opener.open(self.url).read()
return data
def getImage(url):
MAX_TITLE_DESC = 100
MAX_TITLE = 255
uo = URLopener()
file = uo.open(url)
soup = BeautifulSoup.BeautifulSoup(file.read())
file.close()
outImage = Image()
imgTag = soup.find("img", { "class" : "imageWithCaption" })
link = imgTag.get("src", imgTag.get("href", None))
if link:
outImage.url = urllib.basejoin(url, link)
caption = soup.find("div", { "id" : "caption" })
captionTxt = caption.string
#Kuressaare linnus, vaade põhjast (SM F 3761:473 F); Saaremaa Muuseum; Faili nimi:smf_3761_473.jpg
(capPart1, museumName, capPart3) = captionTxt.split(';')
museumName = museumName.strip()
matchItemRef = re.search("^(.+)\((.+?)\)$", capPart1)
if (matchItemRef and matchItemRef.group(2)):
outImage.source = u'[%s %s, %s]' % ( url, museumName, matchItemRef.group(2) )
outImage.source.strip()
mainTable = soup.find("table", {"class" : "data highlighted"})
outDesc = u"<table>\n"
outDesc += getWikiTable(mainTable, outImage)
mainTable = soup.find("table", {"class" : "data"})
outDesc += getWikiTable(mainTable, outImage)
mainTable = soup.find("table", {"class" : "data full_length"})
outDesc += getWikiTable(mainTable, outImage)
outDesc += u"</table>\n"
titleStart = matchItemRef.group(1).strip()
if ( len(titleStart) > MAX_TITLE_DESC ):
#shorten title beginning
titleStart = titleStart[:MAX_TITLE_DESC]
outImage.name = titleStart + u', ' + outImage.accession_number + u'.jpg'
outImage.name = cleanUpTitle( outImage.name )
if ( len(outImage.name) > MAX_TITLE ):
#shorten title
outImage.name = outImage.name[:MAX_TITLE]
outImage.description = '{{et|1=' + outDesc + '}}'
outImage.license = '{{PD-old}}'
##add categories
museumName = museumName.encode('utf_8')
if museumData.get(museumName) and museumData.get(museumName).get('enName'):
museumEnName = museumData.get(museumName).get('enName')
outImage.institution = u'{{Institution:' + museumEnName + u'}}'
museumCat = u'Images from the ' + museumEnName
outImage.categories.append( museumCat )
else:
print "Museum enName not found for %s ! \n" % url
return None
return outImage
def __init__(self, url):
self.opener = URLopener()
self.url = url
self.filename = url.rstrip('/').rsplit('/', 1)[-1]
self.offset = 0
