def main():
desc = ( 'Plotting data' )
usage = ( '%prog [options]\n'
'(type %prog -h for details)' )
op = OptionParser( description=desc, usage=usage )
# Options
op.add_option( '--inputfile', '-i', action="store", \
dest="input_file", help = "Pickled data")
op.add_option( '--outputdir', '-o', action="store", \
dest="output_dir", help = "Directory to store plots")
# Parsing and processing args
options, args = op.parse_args()
args_check = sys.argv[1:]
if len(args_check) != 4:
print 'Something wrong with paramenters. Please check.'
print op.print_help()
sys.exit(1)
# Check and add slash to directory if not there.
output_dir = python_api.check_directory_and_add_slash(options.output_dir)
# Check file, open, read
if os.path.isfile(options.input_file) is True:
fd = open(options.input_file, 'r')
data = pickle.load(fd)
fd.close()
# Plot
saveAsFileName = '' # Add file extension yourself.
plot_title = ''
plot_the_data(data, output_dir, saveAsFileName, plot_title)
def main():
# Parse arguments
parser = OptionParser()
parser.add_option("-i", "--input", dest="input_", help="Input file or directory", metavar="FILE")
parser.add_option("-o", "--output", dest="output_file", help="Output file", metavar="FILE")
parser.add_option("-j", "--jsonfile", dest="json_file", help="Existing json file", metavar="FILE")
(options, args) = parser.parse_args()
# output file
if options.output_file is None:
print "No output file. Abort."
parser.print_help()
sys.exit(1)
# Input file
input_dir = ""
if os.path.isfile(options.input_) is True:
if options.input_ is None:
print "No input file. Abort."
parser.print_help()
sys.exit(1)
else:
report_dict = pickle.load(open(options.input_, 'rb'))
# JSONIZE
jsonize(report_dict)
# # Save json file
# fd = open(options.output_file,'wb')
# json.dump(final_dict,fd)
# fd.close()
# Read directory
elif os.path.isdir(options.input_) is True:
input_dir = python_api.check_directory_and_add_slash(options.input_)
final_map_list,final_map_list_simplified = combine_maps(input_dir, options.output_file)
# Print JSON
# print json.dumps(final_map_list, sort_keys=True, indent=4)
# print json.dumps(final_map_list_simplified, sort_keys=True, indent=4)
# Save json file
# fd = open(options.output_file,'wb')
# json.dump(final_map_list,fd)
# fd.close()
# fd = open(options.output_file + ".simple",'wb')
# json.dump(final_map_list_simplified,fd)
# fd.close()
# Combined if existing JSON file is given
if options.json_file is not None:
combined_json = sum_up_with_existing_json(options.json_file, final_map_list_simplified)
print json.dumps(combined_json , sort_keys=True, indent=4)
fd = open(options.output_file + ".simple_combine_usethis",'wb')
json.dump(combined_json ,fd)
fd.close()
def main():
parser = argparse.ArgumentParser(
description='Script to pull Globus transfer data via REST calls')
parser.add_argument('-c',
dest='config_file',
action='store',
required=True,
help='Configuration file')
parser.add_argument('-o',
dest='output_dir',
action='store',
required=True,
help='Output directory')
parser.add_argument(
'-n',
action='store_true',
help='Add this option if you want to fetch new data from Globus,\
not using the existing pickled data in the output directory'
)
# Parse
args = parser.parse_args()
# Check number of arguments.
if len(sys.argv[1:]) < 4:
print "\nERROR: Wrong number of parameters. -c and -o are required.\n"
parser.print_help()
sys.exit(0)
# Reformat output directory, and check validity.
output_dir = python_api.check_directory_and_add_slash(args.output_dir)
# Check validity of option values
if os.path.isdir(output_dir) is False or os.path.isfile(
args.config_file) is False:
print "\nERROR: Specifid output directory or configuration file does not exist. Abort.\n"
parser.print_help()
sys.exit(-1)
# Read configuration
gconfig = read_configuration_file(args.config_file)
# Define data maps
admin_endpoint_map = {} # { admin endpoint_UUID : endpoint_string_name }
endpoint_id_to_name_map = {
} # { all endpoint_UUID : endpoint_string_name }
endpoint_map_list_of_tasks = {
} # Raw data fetch from Globus. { admin endpoint_UUID : [ tasks ] }
measurement_map_map = {
} # Processed data to produce stats. { admin endpoint_UUID : { field : [ values for each task ] } }
users_set = set() # Set of unique user IDs
# Check if pulled data is present in output_dir.
# If they are all there, and option '-n' was not given, open and load them.
dirs_list = os.listdir(output_dir)
if args.n is False and 'endpoint_map_list_of_tasks.p' in dirs_list:
print 'Using existing maps in directory: ' + str(output_dir)
admin_endpoint_map = pickle.load(
open(output_dir + 'admin_endpoint_map.p', 'rb'))
endpoint_map_list_of_tasks = pickle.load(
open(output_dir + 'endpoint_map_list_of_tasks.p', 'rb'))
# Else, pull new data from Globus via REST call
else:
print 'Pull new Globus data!'
# Get data via REST call
auth_token, transfer_token = get_globus_access_token_client_credentials(
gconfig)
admin_endpoint_map, endpoint_map_list_of_tasks = get_globus_data(
auth_token, transfer_token, gconfig)
# Save data in output directory.
python_api.save_data_as_pickle(endpoint_map_list_of_tasks,
'endpoint_map_list_of_tasks',
output_dir)
python_api.save_data_as_pickle(admin_endpoint_map,
'admin_endpoint_map', output_dir)
# Read raw data from Globus and create processed data dictionaries and sets.
measurement_map_map, users_set, endpoint_id_to_name_map = read_data_and_create_maps(
endpoint_map_list_of_tasks, gconfig)
# Create CSV files (currently fit as input data for Google Chart)
create_csv(admin_endpoint_map, measurement_map_map,
endpoint_id_to_name_map, output_dir, gconfig)
# Create files for html drawing
create_list_of_dtns_for_select(admin_endpoint_map, output_dir)
create_dtn_to_uuid_mapping(admin_endpoint_map, output_dir)
# Print number of unique users
print 'Number of unique users:', len(users_set)
