def _sync_files(self, ftp, ftp_base_dir, expected_remote_files, num_of_expected_remote_files, monitor: Monitor) -> int: sync_files_number = 0 checked_files_number = 0 files_to_download = OrderedDict() file_set_size = 0 for expected_dir_path, expected_filename_dict in expected_remote_files.items(): if monitor.is_cancelled(): raise Cancellation() ftp_dir = ftp_base_dir + '/' + expected_dir_path try: ftp.cwd(ftp_dir) except ftplib.Error: # Note: If we can't CWD to ftp_dir, this usually means, # expected_dir_path may refer to a time range that is not covered remotely. monitor.progress(work=1) continue try: remote_dir_content = ftp.mlsd(facts=['type', 'size', 'modify']) except ftplib.Error: # Note: If we can't MLSD the CWD ftp_dir, we have a problem. monitor.progress(work=1) continue for existing_filename, facts in remote_dir_content: if monitor.is_cancelled(): raise Cancellation() if facts.get('type', None) == 'file' and existing_filename in expected_filename_dict: # update expected_filename_dict with facts of existing_filename expected_filename_dict[existing_filename] = facts file_size = int(facts.get('size', '-1')) if file_size > 0: file_set_size += file_size # TODO (forman, 20160619): put also 'modify' in file_info, to update outdated local files existing_file_info = dict(size=file_size, path=expected_dir_path) files_to_download[existing_filename] = existing_file_info last_cwd = None if files_to_download: dl_stat = _DownloadStatistics(file_set_size) for existing_filename, existing_file_info in files_to_download.items(): checked_files_number += 1 child_monitor = monitor.child(work=1.) if monitor.is_cancelled(): raise Cancellation() if last_cwd is not existing_file_info['path']: ftp.cwd(ftp_base_dir + '/' + existing_file_info['path']) last_cwd = existing_file_info['path'] downloader = FtpDownloader(ftp, existing_filename, existing_file_info, self._file_set_data_store.root_dir, (checked_files_number, num_of_expected_remote_files), child_monitor, dl_stat) result = downloader.start() if DownloadStatus.SUCCESS is result: sync_files_number += 1 return sync_files_number
def _sync_files(self, ftp, ftp_base_dir, expected_remote_files, num_of_expected_remote_files, monitor: Monitor) -> int: sync_files_number = 0 checked_files_number = 0 files_to_download = OrderedDict() file_set_size = 0 for expected_dir_path, expected_filename_dict in expected_remote_files.items(): if monitor.is_cancelled(): raise Cancellation() ftp_dir = ftp_base_dir + '/' + expected_dir_path try: ftp.cwd(ftp_dir) except ftplib.Error: # Note: If we can't CWD to ftp_dir, this usually means, # expected_dir_path may refer to a time range that is not covered remotely. monitor.progress(work=1) continue try: remote_dir_content = ftp.mlsd(facts=['type', 'size', 'modify']) except ftplib.Error: # Note: If we can't MLSD the CWD ftp_dir, we have a problem. monitor.progress(work=1) continue for existing_filename, facts in remote_dir_content: if monitor.is_cancelled(): raise Cancellation() if facts.get('type', None) == 'file' and existing_filename in expected_filename_dict: # update expected_filename_dict with facts of existing_filename expected_filename_dict[existing_filename] = facts file_size = int(facts.get('size', '-1')) if file_size > 0: file_set_size += file_size # TODO (forman, 20160619): put also 'modify' in file_info, to update outdated local files existing_file_info = dict(size=file_size, path=expected_dir_path) files_to_download[existing_filename] = existing_file_info last_cwd = None if files_to_download: dl_stat = _DownloadStatistics(file_set_size) for existing_filename, existing_file_info in files_to_download.items(): checked_files_number += 1 child_monitor = monitor.child(work=1.) if monitor.is_cancelled(): raise Cancellation() if last_cwd is not existing_file_info['path']: ftp.cwd(ftp_base_dir + '/' + existing_file_info['path']) last_cwd = existing_file_info['path'] downloader = FtpDownloader(ftp, existing_filename, existing_file_info, self._file_set_data_store.root_dir, (checked_files_number, num_of_expected_remote_files), child_monitor, dl_stat) result = downloader.start() if DownloadStatus.SUCCESS is result: sync_files_number += 1 return sync_files_number
def _do_json_rpc(web_socket, rpc_request: dict, monitor: Monitor) -> dict: web_socket.write_message(json.dumps(rpc_request)) work_reported = None started = False while True and (monitor is None or not monitor.is_cancelled()): response_str = yield web_socket.read_message() rpc_response = json.loads(response_str) if 'progress' in rpc_response: if monitor: progress = rpc_response['progress'] total = progress.get('total') label = progress.get('label') worked = progress.get('worked') msg = progress.get('message') if not started: monitor.start(label or "start", total_work=total) started = True if started: if worked: if work_reported is None: work_reported = 0.0 work = worked - work_reported work_reported = worked else: work = None monitor.progress(work=work, msg=msg) else: if monitor and started: monitor.done() return rpc_response return {}
def _do_json_rpc(web_socket, rpc_request: dict, monitor: Monitor) -> dict: web_socket.write_message(json.dumps(rpc_request)) work_reported = None started = False while True and (monitor is None or not monitor.is_cancelled()): response_str = yield web_socket.read_message() rpc_response = json.loads(response_str) if 'progress' in rpc_response: if monitor: progress = rpc_response['progress'] total = progress.get('total') label = progress.get('label') worked = progress.get('worked') msg = progress.get('message') if not started: monitor.start(label or "start", total_work=total) started = True if started: if worked: if work_reported is None: work_reported = 0.0 work = worked - work_reported work_reported = worked else: work = None monitor.progress(work=work, msg=msg) else: if monitor and started: monitor.done() return rpc_response return {}
def _fetch_solr_json(base_url, query_args, offset=0, limit=3500, timeout=10, monitor: Monitor = Monitor.NONE): """ Return JSON value read from paginated Solr web-service. """ combined_json_dict = None num_found = -1 # we don't know ahead of time how much request are necessary with monitor.starting("Loading", 10): while True: monitor.progress(work=1) if monitor.is_cancelled(): raise InterruptedError paging_query_args = dict(query_args or {}) paging_query_args.update(offset=offset, limit=limit, format='application/solr+json') url = base_url + '?' + urllib.parse.urlencode(paging_query_args) with urllib.request.urlopen(url, timeout=timeout) as response: json_text = response.read() json_dict = json.loads(json_text.decode('utf-8')) if num_found is -1: num_found = json_dict.get('response', {}).get('numFound', 0) if not combined_json_dict: combined_json_dict = json_dict if num_found < limit: break else: docs = json_dict.get('response', {}).get('docs', []) combined_json_dict.get('response', {}).get('docs', []).extend(docs) if num_found < offset + limit: break offset += limit return combined_json_dict
def data_frame_aggregate(df: DataFrameLike.TYPE, var_names: VarNamesLike.TYPE = None, aggregate_geometry: bool = False, monitor: Monitor = Monitor.NONE) -> pd.DataFrame: """ Aggregate columns into count, mean, median, sum, std, min, and max. Return a new (Geo)DataFrame with a single row containing all aggregated values. Specify whether the geometries of the GeoDataFrame are to be aggregated. All geometries are merged union-like. The return data type will always be the same as the input data type. :param df: The (Geo)DataFrame to be analysed :param var_names: Variables to be aggregated ('None' uses all aggregatable columns) :param aggregate_geometry: Aggregate (union like) the geometry and add it to the resulting GeoDataFrame :param monitor: Monitor for progress bar :return: returns either DataFrame or GeoDataFrame. Keeps input data type """ vns = VarNamesLike.convert(var_names) df_is_geo = isinstance(df, gpd.GeoDataFrame) aggregations = ["count", "mean", "median", "sum", "std", "min", "max"] # Check var names integrity (aggregatable, exists in data frame) types_accepted_for_agg = ['float64', 'int64', 'bool'] agg_columns = list(df.select_dtypes(include=types_accepted_for_agg).columns) if df_is_geo: agg_columns.append('geometry') columns = list(df.columns) if vns is None: vns = agg_columns diff = list(set(vns) - set(columns)) if len(diff) > 0: raise ValidationError('Variable ' + ','.join(diff) + ' not in data frame!') diff = list(set(vns) - set(agg_columns)) if len(diff) > 0: raise ValidationError('Variable(s) ' + ','.join(diff) + ' not aggregatable!') try: df['geometry'] except KeyError as e: raise ValidationError('Variable geometry not in GEO data frame!') from e # Aggregate columns if vns is None: df_buff = df.select_dtypes(include=types_accepted_for_agg).agg(aggregations) else: df_buff = df[vns].select_dtypes(include=types_accepted_for_agg).agg(aggregations) res = {} for n in df_buff.columns: for a in aggregations: val = df_buff[n][a] h = n + '_' + a res[h] = [val] df_agg = pd.DataFrame(res) # Aggregate (union) geometry if GeoDataFrame if df_is_geo and aggregate_geometry: total_work = 100 num_work_rows = 1 + len(df) // total_work with monitor.starting('Aggregating geometry: ', total_work): multi_polygon = shapely.geometry.MultiPolygon() i = 0 for rec in df.geometry: if monitor.is_cancelled(): break # noinspection PyBroadException try: multi_polygon = multi_polygon.union(other=rec) except Exception: pass if i % num_work_rows == 0: monitor.progress(work=1) i += 1 df_agg = gpd.GeoDataFrame(df_agg, geometry=[multi_polygon], crs=df.crs) return df_agg
def _make_local(self, local_ds: LocalDataSource, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE): # local_name = local_ds.name local_id = local_ds.name time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert( var_names) if var_names else None # type: Sequence compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False encoding_update = dict() if compression_enabled: encoding_update.update({ 'zlib': True, 'complevel': compression_level }) if region or var_names: protocol = _ODP_PROTOCOL_OPENDAP else: protocol = _ODP_PROTOCOL_HTTP local_path = os.path.join(local_ds.data_store.data_store_path, local_id) if not os.path.exists(local_path): os.makedirs(local_path) selected_file_list = self._find_files(time_range) if protocol == _ODP_PROTOCOL_OPENDAP: files = self._get_urls_list(selected_file_list, protocol) monitor.start('Sync ' + self.name, total_work=len(files)) for idx, dataset_uri in enumerate(files): child_monitor = monitor.child(work=1) file_name = os.path.basename(dataset_uri) local_filepath = os.path.join(local_path, file_name) time_coverage_start = selected_file_list[idx][1] time_coverage_end = selected_file_list[idx][2] remote_netcdf = None local_netcdf = None try: remote_netcdf = NetCDF4DataStore(dataset_uri) local_netcdf = NetCDF4DataStore(local_filepath, mode='w', persist=True) local_netcdf.set_attributes(remote_netcdf.get_attrs()) remote_dataset = xr.Dataset.load_store(remote_netcdf) process_region = False if region: geo_lat_min = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_min') geo_lat_max = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_max') geo_lon_min = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_min') geo_lon_max = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_max') geo_lat_res = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_resolution') geo_lon_res = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_resolution') if not (isnan(geo_lat_min) or isnan(geo_lat_max) or isnan(geo_lon_min) or isnan(geo_lon_max) or isnan(geo_lat_res) or isnan(geo_lon_res)): process_region = True [lat_min, lon_min, lat_max, lon_max] = region.bounds lat_min = floor( (lat_min - geo_lat_min) / geo_lat_res) lat_max = ceil( (lat_max - geo_lat_min) / geo_lat_res) lon_min = floor( (lon_min - geo_lon_min) / geo_lon_res) lon_max = ceil( (lon_max - geo_lon_min) / geo_lon_res) # TODO (kbernat): check why dataset.sel fails! remote_dataset = remote_dataset.isel( drop=False, lat=slice(lat_min, lat_max), lon=slice(lon_min, lon_max)) geo_lat_max = lat_max * geo_lat_res + geo_lat_min geo_lat_min += lat_min * geo_lat_res geo_lon_max = lon_max * geo_lon_res + geo_lon_min geo_lon_min += lon_min * geo_lon_res if not var_names: var_names = [ var_name for var_name in remote_netcdf.variables.keys() ] var_names.extend([ coord_name for coord_name in remote_dataset.coords.keys() if coord_name not in var_names ]) child_monitor.start(label=file_name, total_work=len(var_names)) for sel_var_name in var_names: var_dataset = remote_dataset.drop([ var_name for var_name in remote_dataset.variables.keys() if var_name != sel_var_name ]) if compression_enabled: var_dataset.variables.get( sel_var_name).encoding.update(encoding_update) local_netcdf.store_dataset(var_dataset) child_monitor.progress(work=1, msg=sel_var_name) if process_region: local_netcdf.set_attribute('geospatial_lat_min', geo_lat_min) local_netcdf.set_attribute('geospatial_lat_max', geo_lat_max) local_netcdf.set_attribute('geospatial_lon_min', geo_lon_min) local_netcdf.set_attribute('geospatial_lon_max', geo_lon_max) finally: if remote_netcdf: remote_netcdf.close() if local_netcdf: local_netcdf.close() local_ds.add_dataset( os.path.join(local_id, file_name), (time_coverage_start, time_coverage_end)) child_monitor.done() else: outdated_file_list = [] for file_rec in selected_file_list: filename, _, _, file_size, url = file_rec dataset_file = os.path.join(local_path, filename) # todo (forman, 20160915): must perform better checks on dataset_file if it is... # ... outdated or incomplete or corrupted. # JSON also includes "checksum" and "checksum_type" fields. if not os.path.isfile(dataset_file) or ( file_size and os.path.getsize(dataset_file) != file_size): outdated_file_list.append(file_rec) if outdated_file_list: with monitor.starting('Sync ' + self.name, len(outdated_file_list)): bytes_to_download = sum( [file_rec[3] for file_rec in outdated_file_list]) dl_stat = _DownloadStatistics(bytes_to_download) file_number = 1 for filename, coverage_from, coverage_to, file_size, url in outdated_file_list: if monitor.is_cancelled(): raise InterruptedError dataset_file = os.path.join(local_path, filename) sub_monitor = monitor.child(work=1.0) # noinspection PyUnusedLocal def reporthook(block_number, read_size, total_file_size): dl_stat.handle_chunk(read_size) if monitor.is_cancelled(): raise InterruptedError sub_monitor.progress(work=read_size, msg=str(dl_stat)) sub_monitor_msg = "file %d of %d" % ( file_number, len(outdated_file_list)) with sub_monitor.starting(sub_monitor_msg, file_size): urllib.request.urlretrieve(url[protocol], filename=dataset_file, reporthook=reporthook) file_number += 1 local_ds.add_dataset(os.path.join(local_id, filename), (coverage_from, coverage_to)) local_ds.save() monitor.done()
def data_frame_aggregate(df: DataFrameLike.TYPE, var_names: VarNamesLike.TYPE = None, aggregate_geometry: bool = False, monitor: Monitor = Monitor.NONE) -> pd.DataFrame: """ Aggregate columns into count, mean, median, sum, std, min, and max. Return a new (Geo)DataFrame with a single row containing all aggregated values. Specify whether the geometries of the GeoDataFrame are to be aggregated. All geometries are merged union-like. The return data type will always be the same as the input data type. :param df: The (Geo)DataFrame to be analysed :param var_names: Variables to be aggregated ('None' uses all aggregatable columns) :param aggregate_geometry: Aggregate (union like) the geometry and add it to the resulting GeoDataFrame :param monitor: Monitor for progress bar :return: returns either DataFrame or GeoDataFrame. Keeps input data type """ vns = VarNamesLike.convert(var_names) df_is_geo = isinstance(df, gpd.GeoDataFrame) aggregations = ["count", "mean", "median", "sum", "std", "min", "max"] # Check var names integrity (aggregatable, exists in data frame) types_accepted_for_agg = ['float64', 'int64', 'bool'] agg_columns = list( df.select_dtypes(include=types_accepted_for_agg).columns) if df_is_geo: agg_columns.append('geometry') columns = list(df.columns) if vns is None: vns = agg_columns diff = list(set(vns) - set(columns)) if len(diff) > 0: raise ValidationError('Variable ' + ','.join(diff) + ' not in data frame!') diff = list(set(vns) - set(agg_columns)) if len(diff) > 0: raise ValidationError('Variable(s) ' + ','.join(diff) + ' not aggregatable!') try: df['geometry'] except KeyError as e: raise ValidationError( 'Variable geometry not in GEO data frame!') from e # Aggregate columns if vns is None: df_buff = df.select_dtypes( include=types_accepted_for_agg).agg(aggregations) else: df_buff = df[vns].select_dtypes( include=types_accepted_for_agg).agg(aggregations) res = {} for n in df_buff.columns: for a in aggregations: val = df_buff[n][a] h = n + '_' + a res[h] = [val] df_agg = pd.DataFrame(res) # Aggregate (union) geometry if GeoDataFrame if df_is_geo and aggregate_geometry: total_work = 100 num_work_rows = 1 + len(df) // total_work with monitor.starting('Aggregating geometry: ', total_work): multi_polygon = shapely.geometry.MultiPolygon() i = 0 for rec in df.geometry: if monitor.is_cancelled(): break # noinspection PyBroadException try: multi_polygon = multi_polygon.union(other=rec) except Exception: pass if i % num_work_rows == 0: monitor.progress(work=1) i += 1 df_agg = gpd.GeoDataFrame(df_agg, geometry=[multi_polygon], crs=df.crs) return df_agg