Exemple #1
0
 def fetch_url(self, url, dest, dirname, args=None):
     """
     - url: File we'll link in.
     - dest: Store the fetched stuff into here
     - dirname: Put the result into a dir with this name, it'll be a subdir of dest
     - args: Additional args to pass to the actual fetcher
     """
     if not os.path.isfile(url):
         self.log.error("File not found: {0}".format(url))
         return False
     filename = os.path.split(url)[-1]
     self.log.debug("Looking for file: {0}".format(filename))
     if os.path.isfile(filename):
         self.log.info("File already exists in source dir: {0}".format(filename))
     else:
         self.log.debug("Symlinking file to source dir.")
         os.symlink(url, os.path.join(os.getcwd(), filename))
     if args.has_key("md5"):
         self.log.debug("Calculating MD5 sum for {0}...".format(filename))
         actual_md5 = utils.md5sum(filename)
         if actual_md5 != args["md5"]:
             self.log.error("MD5 sums do not match!")
             return False
     if utils.is_archive(filename):
         self.log.debug("Unpacking {ar}".format(ar=filename))
         # Move to the correct source location.
         utils.extract_to(filename, dirname)
         # Remove the archive once it has been extracted
         os.remove(filename)
     return True
Exemple #2
0
 def fetch_url(self, url, dest, dirname, args=None):
     """
     - src: URL, without the <type>+ prefix.
     - dest: Store the fetched stuff into here
     - dirname: Put the result into a dir with this name, it'll be a subdir of dest
     - args: Additional args to pass to the actual fetcher
     """
     try:
         self.log.debug("Downloading file: {0}".format(url))
         filename, md5_hash = _download_with_requests(url)
     except IOError as ex:
         self.log.error("Download using requests failed: " + str(ex))
         try:
             self.log.warn("Attempting to download using wget...")
             filename, md5_hash = _download_with_wget(url)
         except PBException as ex:
             self.log.warn(str(ex))
             return False
     self.log.debug("MD5: {0}".format(md5_hash))
     if 'md5' in args and args['md5'] != md5_hash:
         self.log.error("While downloading {fname}: MD5 hashes to not match. Expected {exp}, got {actual}.".format(
             fname=filename, exp=args['md5'], actual=md5_hash
         ))
         return False
     if utils.is_archive(filename):
         # Move archive contents to the correct source location:
         utils.extract_to(filename, dirname)
         # Remove the archive once it has been extracted:
         os.remove(filename)
     return True
Exemple #3
0
 def fetch_url(self, url, dest, dirname, args=None):
     """
     - url: File we'll link in.
     - dest: Store the fetched stuff into here
     - dirname: Put the result into a dir with this name, it'll be a subdir of dest
     - args: Additional args to pass to the actual fetcher
     """
     if not os.path.isfile(url):
         self.log.error("File not found: {0}".format(url))
         return False
     filename = os.path.split(url)[-1]
     self.log.debug("Looking for file: {0}".format(filename))
     if os.path.isfile(filename):
         self.log.info(
             "File already exists in source dir: {0}".format(filename))
     else:
         self.log.debug("Symlinking file to source dir.")
         os.symlink(url, os.path.join(os.getcwd(), filename))
     if args.has_key("md5"):
         self.log.debug("Calculating MD5 sum for {0}...".format(filename))
         actual_md5 = utils.md5sum(filename)
         if actual_md5 != args["md5"]:
             self.log.error("MD5 sums do not match!")
             return False
     if utils.is_archive(filename):
         self.log.debug("Unpacking {ar}".format(ar=filename))
         # Move to the correct source location.
         utils.extract_to(filename, dirname)
         # Remove the archive once it has been extracted
         os.remove(filename)
     return True
Exemple #4
0
 def fetch_url(self, url, dest, dirname, args=None):
     """
     - src: URL, without the <type>+ prefix.
     - dest: Store the fetched stuff into here
     - dirname: Put the result into a dir with this name, it'll be a subdir of dest
     - args: Additional args to pass to the actual fetcher
     """
     try:
         self.log.debug("Downloading file: {0}".format(url))
         filename, md5_hash = _download_with_requests(url)
     except IOError as ex:
         self.log.error("Download using requests failed: " + str(ex))
         try:
             self.log.warn("Attempting to download using wget...")
             filename, md5_hash = _download_with_wget(url)
         except PBException as ex:
             self.log.warn(str(ex))
             return False
     self.log.debug("MD5: {0}".format(md5_hash))
     if 'md5' in args and args['md5'] != md5_hash:
         self.log.error(
             "While downloading {fname}: MD5 hashes to not match. Expected {exp}, got {actual}."
             .format(fname=filename, exp=args['md5'], actual=md5_hash))
         return False
     if utils.is_archive(filename):
         # Move archive contents to the correct source location:
         utils.extract_to(filename, dirname)
         # Remove the archive once it has been extracted:
         os.remove(filename)
     return True
Exemple #5
0
 def fetch_url(self, url, dest, dirname, args=None):
     """
     - src: URL, without the <type>+ prefix.
     - dest: Store the fetched stuff into here
     - dirname: Put the result into a dir with this name, it'll be a subdir of dest
     - args: Additional args to pass to the actual fetcher
     """
     filename = _download(url)
     if utils.is_archive(filename):
         # Move archive contents to the correct source location:
         utils.extract_to(filename, dirname)
         # Remove the archive once it has been extracted:
         os.remove(filename)
     return True
Exemple #6
0
 def fetch_url(self, url, dest, dirname, args=None):
     """
     - src: URL, without the <type>+ prefix.
     - dest: Store the fetched stuff into here
     - dirname: Put the result into a dir with this name, it'll be a subdir of dest
     - args: Additional args to pass to the actual fetcher
     """
     filename = _download(url)
     if utils.is_archive(filename):
         # Move archive contents to the correct source location:
         utils.extract_to(filename, dirname)
         # Remove the archive once it has been extracted:
         os.remove(filename)
     return True