def open_atcf_db(dbname=ATCF_DECKS_DB): '''Open the ATCF Decks Database, create it if it doesn't exist''' # Make sure the directory exists. If the db doesn't exist, # the sqlite3.connect command will create it - which will # fail if the directory doesn't exist. from os.path import dirname as pathdirname from geoips2.filenames.base_paths import make_dirs make_dirs(pathdirname(dbname)) import sqlite3 conn = sqlite3.connect(dbname) conn_cursor = conn.cursor() # Try to create the table - if it already exists, it will just fail # trying to create, pass, and return the already opened db. try: conn_cursor.execute('''CREATE TABLE atcf_deck_stormfiles (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, filename text, last_updated timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, storm_num integer, storm_basin text, start_datetime timestamp, start_lat real, start_lon real, start_vmax real, start_name real, vmax real, end_datetime timestamp)''') # Add in at some point? # storm_start_datetime timestamp, except sqlite3.OperationalError: pass return conn_cursor, conn
# # You should have received a copy of the GNU Lesser General Public License # along with pspbar. If not, see <https://www.gnu.org/licenses/>. # '''(WIFI)Internet-monitoring segments''' from typing import Dict from os.path import join as joinpath from os.path import dirname as pathdirname from subprocess import Popen, PIPE from psutil import net_io_counters from .classes import BarSeg NETCHECK = joinpath(pathdirname(__file__), 'shell_dep', 'netcheck.sh') def ip_addr(_=None) -> Dict[str, str]: '''Create IP ADDRESS string''' color = 0x777777 stdout, stderr = Popen( ['bash', NETCHECK], stdout=PIPE, stderr=PIPE ).communicate() stdout = stdout.decode("utf-8") # print("NET_STATUS:", stdout, stderr) if not stderr: addr = stdout.split("\t")[0] net_type = int(stdout.split("\t")[2]) if net_type & 8:
predictions_filter = parsed_args.predictions_filter if predictions_filter is None: predictions_filter = [] else: filt_parser = lambda x: (x[0], x[1].split(',')) predictions_filter = [ filt_parser(filt.split(':')) for filt in predictions_filter ] predictions_list = construct_path_from( predictions_list, 'predictions', rel_path_to_this_script_dir, predictions_filter=predictions_filter) metrics_list = [ get_metric(metric_name) for metric_name in parsed_args.metrics_list ] reporter = Reporter(output_filepath_template, parsed_args.whole_country) return reporter.report(predictions_list, metrics_list, horizons_list=parsed_args.horizons_list, date_selector=parsed_args.date_selector) if __name__ == '__main__': print('Generated files:' + str( main(sys.argv[1:], rel_path_to_this_script_dir=pathdirname(sys.argv[0]) or '.')))
def old_tcweb_fnames_remove_duplicates(fname, mins_to_remove=10, remove_files=False): # 20201010.222325.WP162020.gmi.GPM.37V.40kts.14p2.1p0.jpg # 20201010.222325.WP162020.gmi.GPM.89H.40kts.14p2.1p0.jpg.yaml matching_fnames = [] removed_fnames = [] saved_fnames = [] ext1 = pathsplitext(fname)[-1] ext2 = pathsplitext(pathsplitext(fname)[0])[-1] ext3 = pathsplitext(pathsplitext(pathsplitext(fname)[0])[0])[-1] if (ext1 == '.jpg') or (ext1 == '.yaml' and ext2 == '.jpg'): LOG.info( 'MATCHES EXT FORMAT. jpg or jpg.yaml. Attempting to remove old_tcweb duplicates' ) else: LOG.info( 'NOT REMOVING DUPLICATES. Not old_tcweb filename, not jpg or jpg.yaml.' ) return [], [] dirname = pathdirname(fname) basename = pathbasename(fname) # 20201010.222325.WP162020.gmi.GPM.37V.40kts.14p2.1p0.jpg parts = basename.split('.') if (len(parts) == 10 and ext1 == '.yaml') or (len(parts) == 9 and ext1 == '.jpg'): LOG.info( 'NOT REMOVING DUPLICATES. Not old_tcweb filename, does not contain 9 or 10 fields.' ) return [], [] try: # 20201010.222325.WP162020.gmi.GPM.37V.40kts.14p2.1p0.jpg yyyymmdd = parts[0] hhmnss = parts[1] stormname = parts[2] sensor = parts[3] platform = parts[4] product = parts[5] intensity = parts[6] coverage = parts[7] res = parts[8] if 'p' not in coverage or 'p' not in res: LOG.info( 'NOT REMOVING DUPLICATES. Not old_tcweb filename, coverage or res not "NNpN.' ) return [], [] if 'kts' not in intensity: LOG.info( 'NOT REMOVING DUPLICATES. Not old_tcweb filename, intensity does not contain "kts".' ) return [], [] except IndexError: LOG.info( 'NOT REMOVING DUPLICATES. Unmatched filename format, incorrect number of . delimited fields' ) return [], [] try: fname_dt = datetime.strptime(yyyymmdd + hhmnss, '%Y%m%d%H%M%S') except ValueError: LOG.info( 'NOT REMOVING DUPLICATES. Unmatched old_tcweb filename format, incorrect date time string.' ) return [], [] timediff = timedelta(minutes=mins_to_remove) for currdt in minrange(fname_dt - timediff, fname_dt + timediff): # 20201010.222325.WP162020.gmi.GPM.37V.40kts.14p2.1p0.jpg # 20201010.222325.WP162020.gmi.GPM.37V.*.*.1p0.jpg* dtstr = currdt.strftime( '{0}/%Y%m%d.%H%M*.{1}.{2}.{3}.{4}.*.*.{5}.jpg*'.format( dirname, stormname, sensor, platform, product, res)) # print(dtstr) matching_fnames += glob(dtstr) max_coverage = 0 for matching_fname in matching_fnames: # 20201010.222325.WP162020.gmi.GPM.37V.40kts.14p2.1p0.jpg parts = pathbasename(matching_fname).split('.') coverage = float(parts[7].replace('p', '.')) max_coverage = max(coverage, max_coverage) gotone = False LOG.info('CHECKING DUPLICATE FILES') for matching_fname in list(set(matching_fnames)): # 20201010.222325.WP162020.gmi.GPM.37V.40kts.14p2.1p0.jpg parts = pathbasename(matching_fname).split('.') coverage = float(parts[7].replace('p', '.')) if coverage < max_coverage or gotone is True: removed_fnames += [matching_fname] # Test it out for a bit first if remove_files is True: LOG.info( 'DELETING DUPLICATE FILE with less coverage %s < %s %s', coverage, max_coverage, matching_fname) osunlink(matching_fname) else: LOG.info( 'TEST DELETING DUPLICATE FILE with less coverage %s < %s %s', coverage, max_coverage, matching_fname) else: if len(matching_fnames) == 1: LOG.info( 'SAVING DUPLICATE FILE (only one!) with max coverage %s %s', max_coverage, matching_fname) else: LOG.info('SAVING DUPLICATE FILE with max coverage %s %s', max_coverage, matching_fname) saved_fnames += [matching_fname] gotone = True return removed_fnames, saved_fnames
def metoctiff_filename_remove_duplicates(fname, mins_to_remove=10, remove_files=False): # 20201010.222325.GPM.gmi.89H.WP162020.14pc.jif.gz # 20201010.222325.GPM.gmi.89H.WP162020.14pc.jif.gz.yaml matching_fnames = [] removed_fnames = [] saved_fnames = [] ext1 = pathsplitext(fname)[-1] ext2 = pathsplitext(pathsplitext(fname)[0])[-1] ext3 = pathsplitext(pathsplitext(pathsplitext(fname)[0])[0])[-1] if (ext1 == '.gz' and ext2 == '.jif') or (ext1 == '.yaml' and ext2 == '.gz' and ext3 == '.jif'): LOG.info( 'MATCHES EXT FORMAT. .jif.gz or .jif.gz.yaml. Attempting to remove metoctiff duplicates' ) else: LOG.info( 'NOT REMOVING DUPLICATES. Not metoctiff filename, not .jif.gz or .jif.gz.yaml.' ) return [], [] dirname = pathdirname(fname) basename = pathbasename(fname) parts = basename.split('.') if (len(parts) == 10 and ext1 == '.yaml') or (len(parts) == 9 and ext1 == '.gz'): LOG.info( 'MATCHES NUMBER FIELDS. 9 or 10 fields. Attempting to remove metoctiff duplicates' ) else: LOG.info( 'NOT REMOVING DUPLICATES. Not metoctiff filename, does not contain 9 or 10 fields.' ) return [], [] try: # 20201010.222325.GPM.gmi.89H.WP162020.14pc.jif.gz yyyymmdd = parts[0] hhmnss = parts[1] platform = parts[2] sensor = parts[3] product = parts[4] stormname = parts[5] coverage = parts[6] if 'pc' not in coverage: LOG.info( 'NOT REMOVING DUPLICATES. Not metoctiff filename, coverage not "NNpc.' ) return [] except IndexError: LOG.info( 'NOT REMOVING DUPLICATES. Unmatched metoctiff filename format, incorrect number of . delimited fields' ) return [], [] try: fname_dt = datetime.strptime(yyyymmdd + hhmnss, '%Y%m%d%H%M%S') except ValueError: LOG.info( 'NOT REMOVING DUPLICATES. Unmatched metoctiff filename format, incorrect date time string.' ) return [], [] timediff = timedelta(minutes=mins_to_remove) for currdt in minrange(fname_dt - timediff, fname_dt + timediff): # 20201010.222325.GPM.gmi.19H.WP162020.14pc.jif.gz # Matches # 20201010.222325.GPM.gmi.19H.WP162020.*.jif.gz* dtstr = currdt.strftime( '{0}/%Y%m%d.%H%M*.{1}.{2}.{3}.{4}.*.jif.gz*'.format( dirname, platform, sensor, product, stormname)) # print(dtstr) matching_fnames += glob(dtstr) max_coverage = 0 for matching_fname in matching_fnames: # 20201010.222325.GPM.gmi.89H.WP162020.14pc.jif.gz parts = pathbasename(matching_fname).split('.') coverage = float(parts[6].replace('pc', '')) max_coverage = max(coverage, max_coverage) gotone = False LOG.info('CHECKING DUPLICATE FILES') for matching_fname in list(set(matching_fnames)): # 20201010.222325.GPM.gmi.89H.WP162020.14pc.jif.gz parts = pathbasename(matching_fname).split('.') coverage = float(parts[6].replace('pc', '')) if coverage < max_coverage or gotone is True: removed_fnames += [matching_fname] # Test it out for a bit first if remove_files is True: LOG.info( 'DELETING DUPLICATE FILE with less coverage %s < %s %s', coverage, max_coverage, matching_fname) osunlink(matching_fname) else: LOG.info( 'TEST DELETING DUPLICATE FILE with less coverage %s < %s %s', coverage, max_coverage, matching_fname) else: saved_fnames += [matching_fname] if len(matching_fnames) == 1: LOG.info( 'SAVING DUPLICATE FILE (only one!) with max coverage %s %s', max_coverage, matching_fname) else: LOG.info('SAVING DUPLICATE FILE with max coverage %s %s', max_coverage, matching_fname) gotone = True return removed_fnames, saved_fnames