def getFileFromURL(url, filename=None, proxyfilename=None): """ Read the content of a URL and copy it into a file. url: the link you would like to retrieve filename: the local filename where the url is saved to. Defaults to the filename in the url proxyfilename: the x509 proxy certificate to be used in case auth is required Return the filename used to save the file or raises ClientException in case of errors (a status attribute is added if the error is an http one). """ parsedurl = urlparse(url) if filename == None: path = parsedurl.path filename = os.path.basename(path) try: opener = urllib.URLopener(key_file=proxyfilename, cert_file=proxyfilename) socket = opener.open(url) status = socket.getcode() # Read the file by chunks instead of all at once, appending each chunk to the final result. # This lowers the memory overhead, which can be a problem with big files. with open(filename, 'a') as f: f.seek(0) f.truncate() while True: piece = socket.read(1024) if not piece: break f.write(piece) except IOError as ioex: msg = "Error while trying to retrieve file from %s: %s" % (url, ioex) msg += "\nMake sure the URL is correct." exc = ClientException(msg) if ioex[0] == 'http error': exc.status = ioex[1] raise exc except Exception as ex: tblogger = logging.getLogger('CRAB3') tblogger.exception(ex) msg = "Unexpected error while trying to retrieve file from %s: %s" % ( url, ex) raise ClientException(msg) if status != 200 and parsedurl.scheme in ['http', 'https']: exc = ClientException( "Unable to retieve the file from %s. HTTP status code %s. HTTP content: %s" % (url, status, socket.info())) exc.status = status raise exc return filename
def getFileFromURL(url, filename = None, proxyfilename = None): """ Read the content of a URL and copy it into a file. url: the link you would like to retrieve filename: the local filename where the url is saved to. Defaults to the filename in the url proxyfilename: the x509 proxy certificate to be used in case auth is required Return the filename used to save the file or raises ClientException in case of errors (a status attribute is added if the error is an http one). """ parsedurl = urlparse(url) if filename == None: path = parsedurl.path filename = os.path.basename(path) try: opener = urllib.URLopener(key_file = proxyfilename, cert_file = proxyfilename) socket = opener.open(url) status = socket.getcode() # Read the file by chunks instead of all at once, appending each chunk to the final result. # This lowers the memory overhead, which can be a problem with big files. with open (filename, 'a') as f: f.seek(0) f.truncate() while True: piece = socket.read(1024) if not piece: break f.write(piece) except IOError as ioex: msg = "Error while trying to retrieve file from %s: %s" % (url, ioex) msg += "\nMake sure the URL is correct." exc = ClientException(msg) if ioex[0] == 'http error': exc.status = ioex[1] raise exc except Exception as ex: tblogger = logging.getLogger('CRAB3') tblogger.exception(ex) msg = "Unexpected error while trying to retrieve file from %s: %s" % (url, ex) raise ClientException(msg) if status != 200 and parsedurl.scheme in ['http', 'https']: exc = ClientException("Unable to retieve the file from %s. HTTP status code %s. HTTP content: %s" % (url, status, socket.info())) exc.status = status raise exc return filename
def getFileFromURL(url, filename=None, proxyfilename=None): """ Read the content of a URL and copy it into a file. url: the link you would like to retrieve filename: the local filename where the url is saved to. Defaults to the filename in the url proxyfilename: the x509 proxy certificate to be used in case auth is required Return the filename used to save the file or raises ClientException in case of errors (a status attribute is added if the error is an http one). """ parsedurl = urlparse(url) if filename == None: path = parsedurl.path filename = os.path.basename(path) try: opener = urllib.URLopener(key_file=proxyfilename, cert_file=proxyfilename) socket = opener.open(url) status = socket.getcode() filestr = socket.read() except IOError as ioex: msg = "Error while trying to retrieve file from %s: %s" % (url, ioex) msg += "\nMake sure the URL is correct." exc = ClientException(msg) if ioex[0] == 'http error': exc.status = ioex[1] raise exc except Exception as ex: tblogger = logging.getLogger('CRAB3') tblogger.exception(ex) msg = "Unexpected error while trying to retrieve file from %s: %s" % ( url, ex) raise ClientException(msg) if status != 200 and parsedurl.scheme in ['http', 'https']: exc = ClientException( "Unable to retieve the file from %s. HTTP status code %s. HTTP content: %s" % (url, status, socket.info())) exc.status = status raise exc with open(filename, 'w') as f: f.write(filestr) return filename
def getFileFromURL(url, filename = None, proxyfilename = None): """ Read the content of a URL and copy it into a file. url: the link you would like to retrieve filename: the local filename where the url is saved to. Defaults to the filename in the url proxyfilename: the x509 proxy certificate to be used in case auth is required Return the filename used to save the file or raises ClientException in case of errors (a status attribute is added if the error is an http one). """ parsedurl = urlparse(url) if filename == None: path = parsedurl.path filename = os.path.basename(path) try: opener = urllib.URLopener(key_file = proxyfilename, cert_file = proxyfilename) socket = opener.open(url) status = socket.getcode() filestr = socket.read() except IOError as ioex: msg = "Error while trying to retrieve file from %s: %s" % (url, ioex) msg += "\nMake sure the URL is correct." exc = ClientException(msg) if ioex[0] == 'http error': exc.status = ioex[1] raise exc except Exception as ex: tblogger = logging.getLogger('CRAB3') tblogger.exception(ex) msg = "Unexpected error while trying to retrieve file from %s: %s" % (url, ex) raise ClientException(msg) if status != 200 and parsedurl.scheme in ['http', 'https']: exc = ClientException("Unable to retieve the file from %s. HTTP status code %s. HTTP content: %s" % (url, status, socket.info())) exc.status = status raise exc with open(filename, 'w') as f: f.write(filestr) return filename