def getDSSfile(arguments,endpoint,destendpoint): if arguments.verbose: print "The list of the corresponding PID is going to be saved in DSSfile." print "Retrieving "+str(arguments.dssfilepath) +" via GridFTP" print "from "+str(endpoint) +" that is "+str(destendpoint[0]) file_list=[] file_list.append(arguments.dssfilepath) json_results=json.dumps(file_list) fo = open("json_file", "w") fo.write(json_results) fo.close() if arguments.verbose: print "Transfer parameters: " print str(arguments.user) print str(destendpoint[0]) print str(arguments.gclocalhost) print str(os.getcwd()) datamover.mover(str(arguments.user), str(destendpoint[0]), str(arguments.gclocalhost), str(os.getcwd()))
def getDSSfile(arguments, endpoint, destendpoint): if arguments.verbose: print "The list of the corresponding PID is going to be saved in DSSfile." print "Retrieving " + str(arguments.dssfilepath) + " via GridFTP" print "from " + str(endpoint) + " that is " + str(destendpoint[0]) file_list = [] file_list.append(arguments.dssfilepath) json_results = json.dumps(file_list) fo = open("json_file", "w") fo.write(json_results) fo.close() if arguments.verbose: print "Transfer parameters: " print str(arguments.user) print str(destendpoint[0]) print str(arguments.gclocalhost) print str(os.getcwd()) datamover.mover(str(arguments.user), str(destendpoint[0]), str(arguments.gclocalhost), str(os.getcwd()))
def main(arguments=None): global kill global stop global interactivity interactivity=None datamover.globality(interactivity) if arguments is None: arguments = sys.argv # Top-level parser parser = argparse.ArgumentParser(description=" Data stager: move a bounce of data inside or outside iRODS via GridFTP. \n The -e options requires both positional arguments.", formatter_class=RawTextHelpFormatter,add_help=True) parser.add_argument('-V', '--version', action='version', version="%(prog)s version 3.1") parser.add_argument('-I', '--interactive', help="Open the DSInteractiveShell", action="store_true") parser.add_argument("-e", "--example", help="a longer description and some usage examples (invoke with \"datastager.py in pid -e\")", action="store_true") # Examples parser.add_argument("-v", "--verbose", help="more informations at run time", action="store_true") parser.add_argument("--ipath", help="your icommands path", action="store") #config file # Turn off help, so we print all options in response to -h parser_cfg_file = argparse.ArgumentParser( add_help=False) parser_cfg_file.add_argument("-c", "--cfg_file", help="Specify config file", metavar="FILE", default="datastager.cfg") arguments, from_file_args = parser_cfg_file.parse_known_args() if arguments.cfg_file: #print "Reading config file..." config = ConfigParser.SafeConfigParser() config.read([arguments.cfg_file]) defaults = dict(config.items("Defaults")) #print defaults else: defaults = { "option":"default" } # # Top-level credentials # parser.set_defaults(**defaults) parser.add_argument("-u", "--username", help="your username on globusonline.org", action="store", dest="user") parser.add_argument("-cert", "--certificate", help="your x509 certificate (pem file)", action="store", dest="cert") parser.add_argument("-key", "--secretekey", help="the key of your certificate", action="store", dest="key") parser.add_argument("-certdir", "--trustedca", help="your trusted CA", action="store", dest="certdir") # Create the subparser subparsers = parser.add_subparsers(help='Directional sub-command help') # # Parser for the "in" direction # subparser_in = subparsers.add_parser('in', help='Stage _in_ is used when moving data into EUDAT.') subparser_in.set_defaults(direction='in') subparsers_in = subparser_in.add_subparsers( help='Stage _in_ sub-command help') subparser_in_issue = subparsers_in.add_parser('issue', help='To issue a transfer.') subparser_in_issue.set_defaults(action='issue') subparser_in_pid = subparsers_in.add_parser('pid', help='To retrieve the PIDs associated to the files you transfereed.') subparser_in_pid.set_defaults(action='pid') subparser_in_details = subparsers_in.add_parser('details', help='To know the status of a transfer.') subparser_in_details.set_defaults(action='details') subparser_in_cancel = subparsers_in.add_parser('cancel', help='To cancel a transfer.') subparser_in_cancel.set_defaults(action='cancel') #issue subparser_in_issue.add_argument("-p", "--path", help="the path of your file", action="store", dest="path") subparser_in_issue.add_argument("-pF", "--pathFile", help="a file containing the path of your file", action="store", dest="pathfile") subparser_in_issue.add_argument("--ss", help="the GridFTP src server identified by its GO endpoint name", action="store", dest="src_site") subparser_in_issue.add_argument("--sd", help="the GridFTP src directory", action="store", dest="src_dir", default="/~/") subparser_in_issue.add_argument("--ds", help="the GridFTP dst server identified by its GO endpoint name", action="store", dest="dst_site") subparser_in_issue.add_argument("--dd", help="the GridFTP dst directory", action="store", dest="dst_dir", default="/~/") #pid subparser_in_pid.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid") subparser_in_pid.add_argument("-RM", "--resolve-mode", help="the way you resolve for source file: iRODS or DSSfile", action="store", dest="rmode") subparser_in_pid.add_argument("-DDS", "--dssfile-default-server", help="the GO endpoint you want to retrieve DSSfile from", action="store", dest="dssfiledefserver") subparser_in_pid.add_argument("-DF", "--dssfile", help="the full iRODS path of DSSfile", action="store", dest="dssfilepath") subparser_in_pid.add_argument("-LE", "--localendpoint", help="the local Globus Connect endpoint", action="store", dest="gclocalhost") #details subparser_in_details.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid", required="true") #cancel subparser_in_cancel.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid", required="true") # # Parser for the "out" direction # subparser_out = subparsers.add_parser('out', help='Stage _out_ is used when moving data outside EUDAT.') subparser_out.set_defaults(direction='out') subparsers_out = subparser_out.add_subparsers( help='Stage _out_ sub-command help') subparser_out_issue = subparsers_out.add_parser('issue', help='To issue a transfer.') subparser_out_issue.set_defaults(action='issue') subparser_out_details = subparsers_out.add_parser('details', help='To know the status of a transfer.') subparser_out_details.set_defaults(action='details') subparser_out_cancel = subparsers_out.add_parser('cancel', help='To cancel a transfer.') subparser_out_cancel.set_defaults(action='cancel') #issue subparsers_out_issues = subparser_out_issue.add_subparsers( help='Stage _out issue_ sub-command help') subparser_out_issue_pid = subparsers_out_issues.add_parser('pid', help='Select data by PIDs') subparser_out_issue_pid.set_defaults(sub_action='pid') subparser_out_issue_url = subparsers_out_issues.add_parser('url', help='Select data by URLs') subparser_out_issue_url.set_defaults(sub_action='url') subparser_out_issue_irods = subparsers_out_issues.add_parser('irods', help='Select data by iRODS URLs') subparser_out_issue_irods.set_defaults(sub_action='irods') #issue -> pid subparser_out_issue_pid.add_argument("-P", "--pid", help="the PID of your data", action="store", dest="pid") subparser_out_issue_pid.add_argument("-PF", "--pid-file", help="the file listing the PID(s) of your data", action="store", dest="pidfile") subparser_out_issue_pid.add_argument("-RM", "--resolve-mode", help="the way you resolve for source file: iRODS or DSSfile", action="store", dest="rmode") subparser_out_issue_pid.add_argument("-DDS", "--dssfile-default-server", help="the GO endpoint you want to retrieve DSSfile from", action="store", dest="dssfiledefserver") subparser_out_issue_pid.add_argument("-DF", "--dssfile", help="the full iRODS path of DSSfile", action="store", dest="dssfilepath") subparser_out_issue_pid.add_argument("-LE", "--localendpoint", help="the local Globus Connect endpoint", action="store", dest="gclocalhost") #issue -> url subparser_out_issue_url.add_argument("-U", "--url", help="the URL of your data", action="store", dest="url") subparser_out_issue_url.add_argument("-UF", "--urlfile", help="the file listing the URL(s) of your data", action="store", dest="urlfile") subparser_out_issue_url.add_argument("-RM", "--resolve-mode", help="the way you resolve for source file: iRODS or DSSfile", action="store", dest="rmode") subparser_out_issue_url.add_argument("-DDS", "--dssfile-default-server", help="the GO endpoint you want to retrieve DSSfile from", action="store", dest="dssfiledefserver") subparser_out_issue_url.add_argument("-DF", "--dssfile", help="the full iRODS path of DSSfile", action="store", dest="dssfilepath") subparser_out_issue_url.add_argument("-LE", "--localendpoint", help="the local Globus Connect endpoint", action="store", dest="gclocalhost") #issue -> irods subparser_out_issue_irods.add_argument("-p", "--path", help="the path of your file (iRODS collection)", action="store", dest="path") subparser_out_issue_irods.add_argument("-pF", "--pathFile", help="a file containing the iRODS path(s) of your file", action="store", dest="pathfile") subparser_out_issue_irods.add_argument("-RM", "--resolve-mode", help="the way you resolve for source file: iRODS or DSSfile", action="store", dest="rmode") subparser_out_issue_irods.add_argument("-DDS", "--dssfile-default-server", help="the GO endpoint you want to retrieve DSSfile from", action="store", dest="dssfiledefserver") subparser_out_issue_irods.add_argument("-DF", "--dssfile", help="the full iRODS path of DSSfile", action="store", dest="dssfilepath") subparser_out_issue_irods.add_argument("-LE", "--localendpoint", help="the local Globus Connect endpoint", action="store", dest="gclocalhost") subparser_out_issue_irods.add_argument("--ss", help="the GridFTP src server identified by its GO endpoint name", action="store", dest="src_site", required="true") #issue -> destination subparser_out_issue.add_argument("--ds", help="the GridFTP dst server identified by its GO endpoint name", action="store", dest="dst_site", required="true") subparser_out_issue.add_argument("--dd", help="the GridFTP dst directory", action="store", dest="dst_dir", default="/~/", required="true") #details subparser_out_details.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid", required="true") #cancel subparser_out_cancel.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid", required="true") # # get everything # arguments = parser.parse_args(from_file_args) if arguments.verbose and arguments.verbose != "None": print arguments if arguments.ipath: ipath=arguments.ipath else: print "The variable ipath must be setted in "+cfg_file # Invoke the detailed help if required if arguments.example: example() ################################################################################## # Start the execution ################################################################################## if not arguments.verbose: os.system('clear') if not arguments.verbose: print "Hello, welcome to data staging!" # Check if the proxy is available and ready check_proxy(arguments) #global kill #global stop kill = False stop = False p = progress_bar_loading() p.start() try: # Parse the arguments argument_parser(arguments) stop = True except KeyboardInterrupt or EOFError: kill = True stop = True #sys.exit(1) ################################################################################## # Actually move the data ################################################################################## api = None datamover.mover(str(arguments.user), str(arguments.src_site), str(arguments.dst_site), str(arguments.dst_dir))
def main(arguments=None): global kill global stop global interactivity interactivity = None datamover.globality(interactivity) if arguments is None: arguments = sys.argv # Top-level parser parser = argparse.ArgumentParser( description= " Data stager: move a bounce of data inside or outside iRODS via GridFTP. \n The -e options requires both positional arguments.", formatter_class=RawTextHelpFormatter, add_help=True) parser.add_argument('-V', '--version', action='version', version="%(prog)s version 3.1") parser.add_argument('-I', '--interactive', help="Open the DSInteractiveShell", action="store_true") parser.add_argument( "-e", "--example", help= "a longer description and some usage examples (invoke with \"datastager.py in pid -e\")", action="store_true") # Examples parser.add_argument("-v", "--verbose", help="more informations at run time", action="store_true") parser.add_argument("--ipath", help="your icommands path", action="store") #config file # Turn off help, so we print all options in response to -h parser_cfg_file = argparse.ArgumentParser(add_help=False) parser_cfg_file.add_argument("-c", "--cfg_file", help="Specify config file", metavar="FILE", default="datastager.cfg") arguments, from_file_args = parser_cfg_file.parse_known_args() if arguments.cfg_file: #print "Reading config file..." config = ConfigParser.SafeConfigParser() config.read([arguments.cfg_file]) defaults = dict(config.items("Defaults")) #print defaults else: defaults = {"option": "default"} # # Top-level credentials # parser.set_defaults(**defaults) parser.add_argument("-u", "--username", help="your username on globusonline.org", action="store", dest="user") parser.add_argument("-cert", "--certificate", help="your x509 certificate (pem file)", action="store", dest="cert") parser.add_argument("-key", "--secretekey", help="the key of your certificate", action="store", dest="key") parser.add_argument("-certdir", "--trustedca", help="your trusted CA", action="store", dest="certdir") # Create the subparser subparsers = parser.add_subparsers(help='Directional sub-command help') # # Parser for the "in" direction # subparser_in = subparsers.add_parser( 'in', help='Stage _in_ is used when moving data into EUDAT.') subparser_in.set_defaults(direction='in') subparsers_in = subparser_in.add_subparsers( help='Stage _in_ sub-command help') subparser_in_issue = subparsers_in.add_parser('issue', help='To issue a transfer.') subparser_in_issue.set_defaults(action='issue') subparser_in_pid = subparsers_in.add_parser( 'pid', help='To retrieve the PIDs associated to the files you transfereed.') subparser_in_pid.set_defaults(action='pid') subparser_in_details = subparsers_in.add_parser( 'details', help='To know the status of a transfer.') subparser_in_details.set_defaults(action='details') subparser_in_cancel = subparsers_in.add_parser( 'cancel', help='To cancel a transfer.') subparser_in_cancel.set_defaults(action='cancel') #issue subparser_in_issue.add_argument("-p", "--path", help="the path of your file", action="store", dest="path") subparser_in_issue.add_argument( "-pF", "--pathFile", help="a file containing the path of your file", action="store", dest="pathfile") subparser_in_issue.add_argument( "--ss", help="the GridFTP src server identified by its GO endpoint name", action="store", dest="src_site") subparser_in_issue.add_argument("--sd", help="the GridFTP src directory", action="store", dest="src_dir", default="/~/") subparser_in_issue.add_argument( "--ds", help="the GridFTP dst server identified by its GO endpoint name", action="store", dest="dst_site") subparser_in_issue.add_argument("--dd", help="the GridFTP dst directory", action="store", dest="dst_dir", default="/~/") #pid subparser_in_pid.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid") subparser_in_pid.add_argument( "-RM", "--resolve-mode", help="the way you resolve for source file: iRODS or DSSfile", action="store", dest="rmode") subparser_in_pid.add_argument( "-DDS", "--dssfile-default-server", help="the GO endpoint you want to retrieve DSSfile from", action="store", dest="dssfiledefserver") subparser_in_pid.add_argument("-DF", "--dssfile", help="the full iRODS path of DSSfile", action="store", dest="dssfilepath") subparser_in_pid.add_argument("-LE", "--localendpoint", help="the local Globus Connect endpoint", action="store", dest="gclocalhost") #details subparser_in_details.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid", required="true") #cancel subparser_in_cancel.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid", required="true") # # Parser for the "out" direction # subparser_out = subparsers.add_parser( 'out', help='Stage _out_ is used when moving data outside EUDAT.') subparser_out.set_defaults(direction='out') subparsers_out = subparser_out.add_subparsers( help='Stage _out_ sub-command help') subparser_out_issue = subparsers_out.add_parser( 'issue', help='To issue a transfer.') subparser_out_issue.set_defaults(action='issue') subparser_out_details = subparsers_out.add_parser( 'details', help='To know the status of a transfer.') subparser_out_details.set_defaults(action='details') subparser_out_cancel = subparsers_out.add_parser( 'cancel', help='To cancel a transfer.') subparser_out_cancel.set_defaults(action='cancel') #issue subparsers_out_issues = subparser_out_issue.add_subparsers( help='Stage _out issue_ sub-command help') subparser_out_issue_pid = subparsers_out_issues.add_parser( 'pid', help='Select data by PIDs') subparser_out_issue_pid.set_defaults(sub_action='pid') subparser_out_issue_url = subparsers_out_issues.add_parser( 'url', help='Select data by URLs') subparser_out_issue_url.set_defaults(sub_action='url') subparser_out_issue_irods = subparsers_out_issues.add_parser( 'irods', help='Select data by iRODS URLs') subparser_out_issue_irods.set_defaults(sub_action='irods') #issue -> pid subparser_out_issue_pid.add_argument("-P", "--pid", help="the PID of your data", action="store", dest="pid") subparser_out_issue_pid.add_argument( "-PF", "--pid-file", help="the file listing the PID(s) of your data", action="store", dest="pidfile") subparser_out_issue_pid.add_argument( "-RM", "--resolve-mode", help="the way you resolve for source file: iRODS or DSSfile", action="store", dest="rmode") subparser_out_issue_pid.add_argument( "-DDS", "--dssfile-default-server", help="the GO endpoint you want to retrieve DSSfile from", action="store", dest="dssfiledefserver") subparser_out_issue_pid.add_argument("-DF", "--dssfile", help="the full iRODS path of DSSfile", action="store", dest="dssfilepath") subparser_out_issue_pid.add_argument( "-LE", "--localendpoint", help="the local Globus Connect endpoint", action="store", dest="gclocalhost") #issue -> url subparser_out_issue_url.add_argument("-U", "--url", help="the URL of your data", action="store", dest="url") subparser_out_issue_url.add_argument( "-UF", "--urlfile", help="the file listing the URL(s) of your data", action="store", dest="urlfile") subparser_out_issue_url.add_argument( "-RM", "--resolve-mode", help="the way you resolve for source file: iRODS or DSSfile", action="store", dest="rmode") subparser_out_issue_url.add_argument( "-DDS", "--dssfile-default-server", help="the GO endpoint you want to retrieve DSSfile from", action="store", dest="dssfiledefserver") subparser_out_issue_url.add_argument("-DF", "--dssfile", help="the full iRODS path of DSSfile", action="store", dest="dssfilepath") subparser_out_issue_url.add_argument( "-LE", "--localendpoint", help="the local Globus Connect endpoint", action="store", dest="gclocalhost") #issue -> irods subparser_out_issue_irods.add_argument( "-p", "--path", help="the path of your file (iRODS collection)", action="store", dest="path") subparser_out_issue_irods.add_argument( "-pF", "--pathFile", help="a file containing the iRODS path(s) of your file", action="store", dest="pathfile") subparser_out_issue_irods.add_argument( "-RM", "--resolve-mode", help="the way you resolve for source file: iRODS or DSSfile", action="store", dest="rmode") subparser_out_issue_irods.add_argument( "-DDS", "--dssfile-default-server", help="the GO endpoint you want to retrieve DSSfile from", action="store", dest="dssfiledefserver") subparser_out_issue_irods.add_argument( "-DF", "--dssfile", help="the full iRODS path of DSSfile", action="store", dest="dssfilepath") subparser_out_issue_irods.add_argument( "-LE", "--localendpoint", help="the local Globus Connect endpoint", action="store", dest="gclocalhost") subparser_out_issue_irods.add_argument( "--ss", help="the GridFTP src server identified by its GO endpoint name", action="store", dest="src_site", required="true") #issue -> destination subparser_out_issue.add_argument( "--ds", help="the GridFTP dst server identified by its GO endpoint name", action="store", dest="dst_site", required="true") subparser_out_issue.add_argument("--dd", help="the GridFTP dst directory", action="store", dest="dst_dir", default="/~/", required="true") #details subparser_out_details.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid", required="true") #cancel subparser_out_cancel.add_argument("-t", "--taskid", help="the taskID of your transfer", action="store", dest="taskid", required="true") # # get everything # arguments = parser.parse_args(from_file_args) if arguments.verbose and arguments.verbose != "None": print arguments if arguments.ipath: ipath = arguments.ipath else: print "The variable ipath must be setted in " + cfg_file # Invoke the detailed help if required if arguments.example: example() ################################################################################## # Start the execution ################################################################################## if not arguments.verbose: os.system('clear') if not arguments.verbose: print "Hello, welcome to data staging!" # Check if the proxy is available and ready check_proxy(arguments) #global kill #global stop kill = False stop = False p = progress_bar_loading() p.start() try: # Parse the arguments argument_parser(arguments) stop = True except KeyboardInterrupt or EOFError: kill = True stop = True #sys.exit(1) ################################################################################## # Actually move the data ################################################################################## api = None datamover.mover(str(arguments.user), str(arguments.src_site), str(arguments.dst_site), str(arguments.dst_dir))