예제 #1
0
    def _cache_locally(self, df: pd.DataFrame, name: str) -> str:
        def clean_cache():
            LOGGER.info('Cleaning cache. This may take some while...')
            # For complete local execution do not clean cache!
            exit(-1)

            shutil.rmtree(self.work_dir)
            Path(self.work_dir).mkdir(parents=True, exist_ok=True)

            self.cache_total, self.cache_used, free = shutil.disk_usage(
                self.work_dir)
            LOGGER.info('Deleted local cache. Using {} of cache'.format(
                self.cache_used / self.cache_total))

        try:
            local_file = store_data(df, self.work_dir, name)
        except IOError:
            clean_cache()
            local_file = store_data(df, self.work_dir, name)

        self.cache_used += os.stat(local_file).st_size
        if self.cache_used / self.cache_total > self.cache_percentage:
            clean_cache()

        return local_file
def store_output_files():
  data.store_data(cname_dir + cname_dict_filename, cnames)
  data.store_data(cname_dir + auth_dict_filename, auths)
  list_data.store_data(cname_dir + cname_filename, list(cname_list))
  list_data.store_data(cname_dir + auth_filename, list(auth_list))
  data.store_data(cname_dir + fail_cname_dict_filename, fail_cnames)
  data.store_data(cname_dir + fail_auth_dict_filename, fail_auths)
  list_data.store_data(cname_dir + fail_cname_filename, list(fail_cname_list))
  list_data.store_data(cname_dir + fail_auth_filename, list(fail_auth_list))
예제 #3
0
def store_output_files():
    data.store_data(output_dir + cname_dict_filename, cnames)
    data.store_data(output_dir + auth_dict_filename, auths)
    list_data.store_data(output_dir + cname_filename, list(cname_list))
    list_data.store_data(output_dir + auth_filename, list(auth_list))
    data.store_data(output_dir + fail_cname_dict_filename, fail_cnames)
    data.store_data(output_dir + fail_auth_dict_filename, fail_auths)
    list_data.store_data(output_dir + fail_cname_filename,
                         list(fail_cname_list))
    list_data.store_data(output_dir + fail_auth_filename, list(fail_auth_list))
def store_output_files():
  data.store_data(ips_dir + ip_dict_filename, ips)
  ip_list = dict_ips_to_list(ips)
  list_data.store_data(ips_dir + ip_filename, ip_list)
예제 #5
0
def store_output_files():
    data.store_data(geo_dir + geo_dict_filename, geo)
    geo_list = geo_dict_to_csv(geo)
    list_data.store_data(geo_dir + geo_filename, geo_list)
예제 #6
0
def store_output_files():
  data.store_data(geo_db_dir + ipinfo_db_filename, located_ips)
예제 #7
0
###################
## Read IP to Geolocation Database
###################
if DEBUG2: print "Read IP to Geolocation Database"

if os.path.exists(geo_db_dir + my_geo_db_filename):
  if DEBUG3: print "  read from my data structure"

  geo_db = data.load_data(geo_db_dir + my_geo_db_filename)

else:
  if DEBUG3: print "  read from orig db"

  geo_db = read_ip2geo_db(geo_db_dir + geo_db_filename)
  data.store_data(geo_db_dir + my_geo_db_filename, geo_db)

located_ips = data.load_data(geo_dir + exist_ip_filename)
if DEBUG3: print "  # records=%d" % (len(geo_db))


# try:
#   address = search_in_db("8.8.8.8", geo_db)
#   print address
# except Exception as e:
#   print "  [1] Search Exception " + type(e) + ": " + e

###################
## Locate IP
###################
if DEBUG2: print "Locate IP"
예제 #8
0
def store_output_files():
  data.store_data(geo_dir + geo_dict_filename, geo)
  geo_list = geo_dict_to_csv(geo)
  list_data.store_data(geo_dir + geo_filename, geo_list)
  data.store_data(geo_dir + exist_ip_filename, located_ips)
def store_output_files():
  data.store_data(geo_dir + geo_dict_filename, geo)
  geo_list = geo_dict_to_csv(geo)
  list_data.store_data(geo_dir + geo_filename, geo_list)
def store_output_files():
    data.store_data(geo_db_dir + ipinfo_db_filename, located_ips)
def store_output_files():
    data.store_data(ips_dir + ip_dict_filename, ips)
    ip_list = dict_ips_to_list(ips)
    list_data.store_data(ips_dir + ip_filename, ip_list)
def store_output_files():
  # ips = merge_ips(ips, data.load_data(ips_dir + ip_dict_filename))
  data.store_data(ips_dir + ip_dict_filename, ips)
  ip_list = dict_ips_to_list(ips)
  list_data.store_data(ips_dir + ip_filename, ip_list)
예제 #13
0
파일: eval_dy.py 프로젝트: personx000/v-cdn
        plt.savefig(os.path.join(args.evalf, 'cor_raw_%d.pdf' % idx_rel))
        plt.show()

# store data for plotting
if args.env in ['Ball']:
    # edge_acc_over_time: n_roll x n_timestep

    record_names = ['edge_acc_over_time', 'edge_cor_over_time', 'fwd_loss_mse']
    if args.baseline == 1:
        record_path = os.path.join(args.evalf,
                                   'rec_%d_baseline.h5' % args.n_kp)
    else:
        record_path = os.path.join(args.evalf, 'rec_%d.h5' % args.n_kp)

    store_data(
        record_names,
        [edge_acc_over_time_record, edge_cor_over_time_record, fwd_loss_mse],
        record_path)

    print()
    print('Edge Accuracy')
    print('%.2f%%, std: %.6f' %
          (np.mean(edge_acc_over_time_record[:, -1]) * 100.,
           np.std(edge_acc_over_time_record[:, -1])))

    print()
    print('Correlation on Attributes')
    for i in range(len(edge_cor_over_time_record)):
        print('#%d:' % i, edge_cor_over_time_record[i][-1])

elif args.env in ['Cloth']: