def main(): desc = ( 'Plotting data' ) usage = ( '%prog [options]\n' '(type %prog -h for details)' ) op = OptionParser( description=desc, usage=usage ) # Options op.add_option( '--inputfile', '-i', action="store", \ dest="input_file", help = "Pickled data") op.add_option( '--outputdir', '-o', action="store", \ dest="output_dir", help = "Directory to store plots") # Parsing and processing args options, args = op.parse_args() args_check = sys.argv[1:] if len(args_check) != 4: print 'Something wrong with paramenters. Please check.' print op.print_help() sys.exit(1) # Check and add slash to directory if not there. output_dir = python_api.check_directory_and_add_slash(options.output_dir) # Check file, open, read if os.path.isfile(options.input_file) is True: fd = open(options.input_file, 'r') data = pickle.load(fd) fd.close() # Plot saveAsFileName = '' # Add file extension yourself. plot_title = '' plot_the_data(data, output_dir, saveAsFileName, plot_title)
def main(): # Parse arguments parser = OptionParser() parser.add_option("-i", "--input", dest="input_", help="Input file or directory", metavar="FILE") parser.add_option("-o", "--output", dest="output_file", help="Output file", metavar="FILE") parser.add_option("-j", "--jsonfile", dest="json_file", help="Existing json file", metavar="FILE") (options, args) = parser.parse_args() # output file if options.output_file is None: print "No output file. Abort." parser.print_help() sys.exit(1) # Input file input_dir = "" if os.path.isfile(options.input_) is True: if options.input_ is None: print "No input file. Abort." parser.print_help() sys.exit(1) else: report_dict = pickle.load(open(options.input_, 'rb')) # JSONIZE jsonize(report_dict) # # Save json file # fd = open(options.output_file,'wb') # json.dump(final_dict,fd) # fd.close() # Read directory elif os.path.isdir(options.input_) is True: input_dir = python_api.check_directory_and_add_slash(options.input_) final_map_list,final_map_list_simplified = combine_maps(input_dir, options.output_file) # Print JSON # print json.dumps(final_map_list, sort_keys=True, indent=4) # print json.dumps(final_map_list_simplified, sort_keys=True, indent=4) # Save json file # fd = open(options.output_file,'wb') # json.dump(final_map_list,fd) # fd.close() # fd = open(options.output_file + ".simple",'wb') # json.dump(final_map_list_simplified,fd) # fd.close() # Combined if existing JSON file is given if options.json_file is not None: combined_json = sum_up_with_existing_json(options.json_file, final_map_list_simplified) print json.dumps(combined_json , sort_keys=True, indent=4) fd = open(options.output_file + ".simple_combine_usethis",'wb') json.dump(combined_json ,fd) fd.close()
def main(): parser = argparse.ArgumentParser( description='Script to pull Globus transfer data via REST calls') parser.add_argument('-c', dest='config_file', action='store', required=True, help='Configuration file') parser.add_argument('-o', dest='output_dir', action='store', required=True, help='Output directory') parser.add_argument( '-n', action='store_true', help='Add this option if you want to fetch new data from Globus,\ not using the existing pickled data in the output directory' ) # Parse args = parser.parse_args() # Check number of arguments. if len(sys.argv[1:]) < 4: print "\nERROR: Wrong number of parameters. -c and -o are required.\n" parser.print_help() sys.exit(0) # Reformat output directory, and check validity. output_dir = python_api.check_directory_and_add_slash(args.output_dir) # Check validity of option values if os.path.isdir(output_dir) is False or os.path.isfile( args.config_file) is False: print "\nERROR: Specifid output directory or configuration file does not exist. Abort.\n" parser.print_help() sys.exit(-1) # Read configuration gconfig = read_configuration_file(args.config_file) # Define data maps admin_endpoint_map = {} # { admin endpoint_UUID : endpoint_string_name } endpoint_id_to_name_map = { } # { all endpoint_UUID : endpoint_string_name } endpoint_map_list_of_tasks = { } # Raw data fetch from Globus. { admin endpoint_UUID : [ tasks ] } measurement_map_map = { } # Processed data to produce stats. { admin endpoint_UUID : { field : [ values for each task ] } } users_set = set() # Set of unique user IDs # Check if pulled data is present in output_dir. # If they are all there, and option '-n' was not given, open and load them. dirs_list = os.listdir(output_dir) if args.n is False and 'endpoint_map_list_of_tasks.p' in dirs_list: print 'Using existing maps in directory: ' + str(output_dir) admin_endpoint_map = pickle.load( open(output_dir + 'admin_endpoint_map.p', 'rb')) endpoint_map_list_of_tasks = pickle.load( open(output_dir + 'endpoint_map_list_of_tasks.p', 'rb')) # Else, pull new data from Globus via REST call else: print 'Pull new Globus data!' # Get data via REST call auth_token, transfer_token = get_globus_access_token_client_credentials( gconfig) admin_endpoint_map, endpoint_map_list_of_tasks = get_globus_data( auth_token, transfer_token, gconfig) # Save data in output directory. python_api.save_data_as_pickle(endpoint_map_list_of_tasks, 'endpoint_map_list_of_tasks', output_dir) python_api.save_data_as_pickle(admin_endpoint_map, 'admin_endpoint_map', output_dir) # Read raw data from Globus and create processed data dictionaries and sets. measurement_map_map, users_set, endpoint_id_to_name_map = read_data_and_create_maps( endpoint_map_list_of_tasks, gconfig) # Create CSV files (currently fit as input data for Google Chart) create_csv(admin_endpoint_map, measurement_map_map, endpoint_id_to_name_map, output_dir, gconfig) # Create files for html drawing create_list_of_dtns_for_select(admin_endpoint_map, output_dir) create_dtn_to_uuid_mapping(admin_endpoint_map, output_dir) # Print number of unique users print 'Number of unique users:', len(users_set)