def fetch_url(self, url, dest, dirname, args=None):
"""
- url: File we'll link in.
- dest: Store the fetched stuff into here
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
if not os.path.isfile(url):
self.log.error("File not found: {0}".format(url))
return False
filename = os.path.split(url)[-1]
self.log.debug("Looking for file: {0}".format(filename))
if os.path.isfile(filename):
self.log.info("File already exists in source dir: {0}".format(filename))
else:
self.log.debug("Symlinking file to source dir.")
os.symlink(url, os.path.join(os.getcwd(), filename))
if args.has_key("md5"):
self.log.debug("Calculating MD5 sum for {0}...".format(filename))
actual_md5 = utils.md5sum(filename)
if actual_md5 != args["md5"]:
self.log.error("MD5 sums do not match!")
return False
if utils.is_archive(filename):
self.log.debug("Unpacking {ar}".format(ar=filename))
# Move to the correct source location.
utils.extract_to(filename, dirname)
# Remove the archive once it has been extracted
os.remove(filename)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
- src: URL, without the <type>+ prefix.
- dest: Store the fetched stuff into here
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
try:
self.log.debug("Downloading file: {0}".format(url))
filename, md5_hash = _download_with_requests(url)
except IOError as ex:
self.log.error("Download using requests failed: " + str(ex))
try:
self.log.warn("Attempting to download using wget...")
filename, md5_hash = _download_with_wget(url)
except PBException as ex:
self.log.warn(str(ex))
return False
self.log.debug("MD5: {0}".format(md5_hash))
if 'md5' in args and args['md5'] != md5_hash:
self.log.error("While downloading {fname}: MD5 hashes to not match. Expected {exp}, got {actual}.".format(
fname=filename, exp=args['md5'], actual=md5_hash
))
return False
if utils.is_archive(filename):
# Move archive contents to the correct source location:
utils.extract_to(filename, dirname)
# Remove the archive once it has been extracted:
os.remove(filename)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
- url: File we'll link in.
- dest: Store the fetched stuff into here
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
if not os.path.isfile(url):
self.log.error("File not found: {0}".format(url))
return False
filename = os.path.split(url)[-1]
self.log.debug("Looking for file: {0}".format(filename))
if os.path.isfile(filename):
self.log.info(
"File already exists in source dir: {0}".format(filename))
else:
self.log.debug("Symlinking file to source dir.")
os.symlink(url, os.path.join(os.getcwd(), filename))
if args.has_key("md5"):
self.log.debug("Calculating MD5 sum for {0}...".format(filename))
actual_md5 = utils.md5sum(filename)
if actual_md5 != args["md5"]:
self.log.error("MD5 sums do not match!")
return False
if utils.is_archive(filename):
self.log.debug("Unpacking {ar}".format(ar=filename))
# Move to the correct source location.
utils.extract_to(filename, dirname)
# Remove the archive once it has been extracted
os.remove(filename)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
- src: URL, without the <type>+ prefix.
- dest: Store the fetched stuff into here
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
try:
self.log.debug("Downloading file: {0}".format(url))
filename, md5_hash = _download_with_requests(url)
except IOError as ex:
self.log.error("Download using requests failed: " + str(ex))
try:
self.log.warn("Attempting to download using wget...")
filename, md5_hash = _download_with_wget(url)
except PBException as ex:
self.log.warn(str(ex))
return False
self.log.debug("MD5: {0}".format(md5_hash))
if 'md5' in args and args['md5'] != md5_hash:
self.log.error(
"While downloading {fname}: MD5 hashes to not match. Expected {exp}, got {actual}."
.format(fname=filename, exp=args['md5'], actual=md5_hash))
return False
if utils.is_archive(filename):
# Move archive contents to the correct source location:
utils.extract_to(filename, dirname)
# Remove the archive once it has been extracted:
os.remove(filename)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
- src: URL, without the <type>+ prefix.
- dest: Store the fetched stuff into here
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
filename = _download(url)
if utils.is_archive(filename):
# Move archive contents to the correct source location:
utils.extract_to(filename, dirname)
# Remove the archive once it has been extracted:
os.remove(filename)
return True
def fetch_url(self, url, dest, dirname, args=None):
"""
- src: URL, without the <type>+ prefix.
- dest: Store the fetched stuff into here
- dirname: Put the result into a dir with this name, it'll be a subdir of dest
- args: Additional args to pass to the actual fetcher
"""
filename = _download(url)
if utils.is_archive(filename):
# Move archive contents to the correct source location:
utils.extract_to(filename, dirname)
# Remove the archive once it has been extracted:
os.remove(filename)
return True