def _get_default_uri(cls): """Returns the URL for the latest GOES data.""" now = datetime.datetime.utcnow() time_range = TimeRange(datetime.datetime(now.year, now.month, now.day), now) url_does_exist = net.url_exists(cls._get_url_for_date_range(time_range)) while not url_does_exist: time_range = TimeRange(time_range.start-datetime.timedelta(days=1), time_range.start) url_does_exist = net.url_exists(cls._get_url_for_date_range(time_range)) return cls._get_url_for_date_range(time_range)
def _get_default_uri(cls): """Returns the URL for the latest GOES data.""" now = datetime.datetime.utcnow() time_range = TimeRange(datetime.datetime(now.year, now.month, now.day), now) url_does_exist = net.url_exists( cls._get_url_for_date_range(time_range)) while not url_does_exist: time_range = TimeRange( time_range.start - datetime.timedelta(days=1), time_range.start) url_does_exist = net.url_exists( cls._get_url_for_date_range(time_range)) return cls._get_url_for_date_range(time_range)
def download_sample_data(progress=True): """ Download the sample data. Parameters ---------- progress: bool Show a progress bar during download Returns ------- None """ number_of_files_fetched = 0 print("Downloading sample files to " + sampledata_dir) for base_url in _base_urls: for file_name in _files.itervalues(): full_file_name = file_name[0] + file_name[1] if url_exists(os.path.join(base_url, full_file_name)): f = download_file(os.path.join(base_url, full_file_name)) real_name, ext = os.path.splitext(full_file_name) if file_name[1] == '.zip': print("Unpacking: %s" % real_name) with ZipFile(f, 'r') as zip_file: zip_file.extract(real_name, sampledata_dir) remove(f) else: # move files to the data directory move(f, os.path.join(sampledata_dir, file_name[0])) # increment the number of files obtained to check later number_of_files_fetched += 1 if number_of_files_fetched < len(_files.keys()): raise URLError("Could not download all samples files. Problem with accessing sample data servers.")
def download_sample_data(progress=True, overwrite=True, timeout=None): """ Download the sample data. Parameters ---------- progress: `bool` Show a progress bar during download overwrite: `bool` If exist overwrites the downloaded sample data. timeout: `float` The timeout in seconds. If `None` the default timeout is used from `astropy.utils.data.Conf.remote_timeout`. Returns ------- None """ # Creating the directory for sample files to be downloaded sampledata_dir = get_and_create_sample_dir() number_of_files_fetched = 0 print("Downloading sample files to {}".format(sampledata_dir)) for file_name in six.itervalues(_files): if not overwrite: if os.path.isfile(os.path.join(sampledata_dir, file_name[0])): number_of_files_fetched += 1 continue for base_url in _base_urls: full_file_name = file_name[0] + file_name[1] try: exists = url_exists(os.path.join(base_url, full_file_name)) if exists: f = download_file(os.path.join(base_url, full_file_name)) real_name, ext = os.path.splitext(full_file_name) if file_name[1] == '.zip': print("Unpacking: {}".format(real_name)) with ZipFile(f, 'r') as zip_file: zip_file.extract(real_name, sampledata_dir) os.remove(f) else: # move files to the data directory move(f, os.path.join(sampledata_dir, file_name[0])) # increment the number of files obtained to check later number_of_files_fetched += 1 break except (socket.error, socket.timeout) as e: warnings.warn("Download failed with error {}. \n" "Retrying with different mirror.".format(e)) if number_of_files_fetched < len(list(_files.keys())): raise URLError("Could not download all samples files." "Problem with accessing sample data servers.")
def download_sample_data(progress=True, overwrite=True, timeout=None): """ Download the sample data. Parameters ---------- progress: `bool` Show a progress bar during download overwrite: `bool` If exist overwrites the downloaded sample data. timeout: `float` The timeout in seconds. If `None` the default timeout is used from `astropy.utils.data.Conf.remote_timeout`. Returns ------- None """ number_of_files_fetched = 0 print("Downloading sample files to {}".format(sampledata_dir)) for file_name in six.itervalues(_files): if not overwrite: if os.path.isfile(os.path.join(sampledata_dir, file_name[0])): number_of_files_fetched += 1 continue for base_url in _base_urls: full_file_name = file_name[0] + file_name[1] try: exists = url_exists(os.path.join(base_url, full_file_name)) if exists: f = download_file(os.path.join(base_url, full_file_name)) real_name, ext = os.path.splitext(full_file_name) if file_name[1] == '.zip': print("Unpacking: {}".format(real_name)) with ZipFile(f, 'r') as zip_file: zip_file.extract(real_name, sampledata_dir) os.remove(f) else: # move files to the data directory move(f, os.path.join(sampledata_dir, file_name[0])) # increment the number of files obtained to check later number_of_files_fetched += 1 break except (socket.error, socket.timeout) as e: warnings.warn("Download failed with error {}. \n Retrying with different mirror.".format(e)) if number_of_files_fetched < len(list(_files.keys())): raise URLError("Could not download all samples files. Problem with accessing sample data servers.")
def download_sample_data(): """ Download the sample data. Parameters ---------- None Returns ------- None """ print("Downloading sample fiss files to {}".format(sampledir)) for f in files: if url_exists(url+f): df = download_file(url+f) move(df,os.path.join(sampledir,f))
def download_sample_data(progress=True, overwrite=True): """ Download the sample data. Parameters ---------- progress: bool Show a progress bar during download overwrite: bool If exist overwrites the downloaded sample data. Returns ------- None """ number_of_files_fetched = 0 print("Downloading sample files to " + sampledata_dir) for file_name in _files.itervalues(): if not overwrite: if os.path.isfile(os.path.join(sampledata_dir, file_name[0])): number_of_files_fetched += 1 continue for base_url in _base_urls: full_file_name = file_name[0] + file_name[1] print(full_file_name) if url_exists(os.path.join(base_url, full_file_name)): f = download_file(os.path.join(base_url, full_file_name)) real_name, ext = os.path.splitext(full_file_name) if file_name[1] == '.zip': print("Unpacking: %s" % real_name) with ZipFile(f, 'r') as zip_file: zip_file.extract(real_name, sampledata_dir) remove(f) else: # move files to the data directory move(f, os.path.join(sampledata_dir, file_name[0])) # increment the number of files obtained to check later number_of_files_fetched += 1 break if number_of_files_fetched < len(_files.keys()): raise URLError( "Could not download all samples files. Problem with accessing sample data servers." )
def download_sample_data(overwrite=True,**kwargs): """ Download the sample data. Parameters ---------- progress: bool Show a progress bar during download overwrite: bool If exist overwrites the downloaded sample data. Returns ------- None """ number_of_files_fetched = 0 print("Downloading sample files to {}".format(sampledata_dir)) for file_name in six.itervalues(_files): if not overwrite: if os.path.isfile(os.path.join(sampledata_dir, file_name[0])): number_of_files_fetched += 1 continue for base_url in _base_urls: full_file_name = file_name[0] + file_name[1] if url_exists(os.path.join(base_url, full_file_name)): f = download_file(os.path.join(base_url, full_file_name)) real_name, ext = os.path.splitext(full_file_name) if file_name[1] == '.zip': print("Unpacking: {}".format(real_name)) with ZipFile(f, 'r') as zip_file: zip_file.extract(real_name, sampledata_dir) os.remove(f) else: # move files to the data directory move(f, os.path.join(sampledata_dir, file_name[0])) # increment the number of files obtained to check later number_of_files_fetched += 1 break if number_of_files_fetched < len(list(_files.keys())): raise URLError("Could not download all samples files. Problem with accessing sample data servers.")
def get_sample_file(filename, url_list, show_progress=True, overwrite=False, timeout=None): """ Downloads a sample file. Will download a sample data file and move it to the sample data directory. Also, uncompresses zip files if necessary. Returns the local file if exists. Parameters ---------- filename: `str` Name of the file url_list: `str` or `list` urls where to look for the file show_progress: `bool` Show a progress bar during download overwrite: `bool` If True download and overwrite an existing file. timeout: `float` The timeout in seconds. If `None` the default timeout is used from `astropy.utils.data.Conf.remote_timeout`. Returns ------- result: `str` The local path of the file. None if it failed. """ # Creating the directory for sample files to be downloaded sampledata_dir = get_and_create_sample_dir() if filename[-3:] == 'zip': uncompressed_filename = filename[:-4] else: uncompressed_filename = filename # check if the (uncompressed) file exists if not overwrite and os.path.isfile( os.path.join(sampledata_dir, uncompressed_filename)): return os.path.join(sampledata_dir, uncompressed_filename) else: # check each provided url to find the file for base_url in url_list: online_filename = filename if base_url.count('github'): online_filename += '?raw=true' try: url = urljoin(base_url, online_filename) exists = url_exists(url) if exists: f = download_file(os.path.join(base_url, online_filename), show_progress=show_progress, timeout=timeout) real_name, ext = os.path.splitext(f) if ext == '.zip': print("Unpacking: {}".format(real_name)) with ZipFile(f, 'r') as zip_file: unzipped_f = zip_file.extract( real_name, sampledata_dir) os.remove(f) move( unzipped_f, os.path.join(sampledata_dir, uncompressed_filename)) return os.path.join(sampledata_dir, uncompressed_filename) else: # move files to the data directory move( f, os.path.join(sampledata_dir, uncompressed_filename)) return os.path.join(sampledata_dir, uncompressed_filename) except (socket.error, socket.timeout) as e: warnings.warn( "Download failed with error {}. Retrying with different mirror." .format(e), SunpyUserWarning) # if reach here then file has not been downloaded. warnings.warn("File {} not found.".format(filename), SunpyUserWarning) return None
def get_sample_file(filename, url_list, show_progress=True, overwrite=False, timeout=None): """ Downloads a sample file. Will download a sample data file and move it to the sample data directory. Also, uncompresses zip files if necessary. Returns the local file if exists. Parameters ---------- filename: `str` Name of the file url_list: `str` or `list` urls where to look for the file show_progress: `bool` Show a progress bar during download overwrite: `bool` If True download and overwrite an existing file. timeout: `float` The timeout in seconds. If `None` the default timeout is used from `astropy.utils.data.Conf.remote_timeout`. Returns ------- result: `str` The local path of the file. None if it failed. """ # Creating the directory for sample files to be downloaded sampledata_dir = get_and_create_sample_dir() if filename[-3:] == 'zip': uncompressed_filename = filename[:-4] else: uncompressed_filename = filename # check if the (uncompressed) file exists if not overwrite and os.path.isfile(os.path.join(sampledata_dir, uncompressed_filename)): return os.path.join(sampledata_dir, uncompressed_filename) else: # check each provided url to find the file for base_url in url_list: online_filename = filename if base_url.count('github'): online_filename += '?raw=true' try: url = urljoin(base_url, online_filename) exists = url_exists(url) if exists: f = download_file(os.path.join(base_url, online_filename), show_progress=show_progress, timeout=timeout) real_name, ext = os.path.splitext(f) if ext == '.zip': print("Unpacking: {}".format(real_name)) with ZipFile(f, 'r') as zip_file: unzipped_f = zip_file.extract(real_name, sampledata_dir) os.remove(f) move(unzipped_f, os.path.join(sampledata_dir, uncompressed_filename)) return os.path.join(sampledata_dir, uncompressed_filename) else: # move files to the data directory move(f, os.path.join(sampledata_dir, uncompressed_filename)) return os.path.join(sampledata_dir, uncompressed_filename) except (socket.error, socket.timeout) as e: warnings.warn("Download failed with error {}. \n" "Retrying with different mirror.".format(e)) # if reach here then file has not been downloaded. warnings.warn("File {} not found.".format(filename)) return None