def stop_del(prof, args, **kwargs): """Call --del action on a server""" # retreive the study export before it would be removed study_prof = get_study_export(prof) jobid = prof['jobid'][0] jobname = prof['nomjob'][0] mode = prof['mode'][0] node = prof['noeud'][0] targs = (None, jobid, jobname, mode, node) signal = kwargs.get('signal', magic.run['signal']) # read server informations serv = build_server_from_profile(prof, TYPES.EXEC) magic.log.info( _(u"ask the server to cancel the job and remove its " "files from 'flasheur'")) iret, output, err = call_generic_service("del", serv, prof, targs, {'signal': signal}) # stop here if signal!=KILL or the study id have not been retreived if signal != 'KILL' or study_prof is None: return iret # remove remote repository studyid = study_prof['studyid'][0] scopy = build_server_from_profile(study_prof, TYPES.COPY_FROM, jobid=studyid) scopy.delete_proxy_dir() return iret
def create_issue(prof, args, **kwargs): """Call --create_issue on a server.""" # put the files on the server run = magic.run num_job = kwargs.get('num_job', run['num_job']) jobid = '%s-%s' % (num_job, "create_issue") # read server informations sexec = build_server_from_profile(prof, TYPES.EXEC) scopy = build_server_from_profile(prof, TYPES.COPY_TO, jobid=jobid) # read issue file issue_file = args[0] content = open(issue_file, 'r').read() dinf = parse_issue_file(content) scopy.copyto(issue_file) rfile = scopy.get_remote_filename(issue_file) # should we copy data files ? profname = "no_attachment" if dinf.get('FICASS') is not None: fprof = osp.join(run['tmp_user'], "%s.export" % jobid) iret, remote_prof = copy_datafiles_on_server(prof, jobid, fprof) profname = remote_prof.get_filename() if iret != 0: return iret targs = (None, rfile, profname) iret, output, err = call_generic_service("create_issue", sexec, prof, targs) scopy.delete_proxy_dir() return iret
def insert_in_db(prof, args, **kwargs): """Call --insert_in_db on a server.""" run = magic.run num_job = kwargs.get('num_job', run['num_job']) jobid = '%s-%s' % (num_job, "insert_in_db") # read server informations sexec = build_server_from_profile(prof, TYPES.EXEC) scopy = build_server_from_profile(prof, TYPES.COPY_TO, jobid=jobid) # copy data files #XXX should result files be copied too ? #XXX should be run in foreground to keep display fprof = osp.join(run['tmp_user'], "%s.export" % jobid) iret, remote_prof = copy_datafiles_on_server(prof, jobid, fprof) if iret != 0: return iret targs = ( None, remote_prof.get_filename(), ) iret, output, err = call_generic_service("insert_in_db", sexec, prof, targs) scopy.delete_proxy_dir() return iret
def copy_datafiles_on_server(prof, studyid, fprof): """Copy data files on the server, relocate the export and write it (locally) into 'fprof'. Return the relocated export.""" # read server informations sexec = build_server_from_profile(prof, TYPES.EXEC) scopy = build_server_from_profile(prof, TYPES.COPY_TO, jobid=studyid) forig = prof.get_filename() #peter.zhang for remote connection forig = forig.replace('\\', '/') magic.log.debug("original export name is %s", forig) magic.run.DBG("original export :\n%s" % repr(prof), all=True) run_on_localhost = sexec.is_localhost() remote_prof = prof.copy() if not run_on_localhost: remote_prof.relocate(local_full_host, scopy.get_proxy_dir(), convert=unique_basename) remote_prof.relocate(sexec.host, convert=unique_basename) remote_prof['studyid'] = studyid remote_prof.set_filename(fprof) remote_prof.WriteExportTo(fprof) magic.run.DBG("remote export :\n%s" % repr(remote_prof), all=True) # copy data files # - local_data : files which are on localhost and to copy on the compute server all_data = prof.get_data() if not run_on_localhost: local_data = all_data.get_on_serv(local_full_host) else: local_data = [] # - remote_data : these files are already on the server, use them directly remote_data = all_data.get_on_serv(sexec.host, sexec.user).topath() # - foreign_data : if they are on a foreign server (not localhost or # compute server), we can : # - first copy them locally, and send them to the compute server. # - or try to copy them from the compute server (<<< let this choice right now). all = set(all_data.topath()) if local_data: all.difference_update(local_data.topath()) all.difference_update(remote_data) if len(all) > 0: magic.log.warn( _(u"files on a third host may be unavailable " "for calculation : %s"), all) # copy files on the server magic.log.info(_(u"copy export files...")) iret = scopy.copyto(fprof, forig) magic.log.info(_(u"copy data files...")) if local_data: local_nom, local_other = local_data.get_type('nom', with_completion=True) iret = scopy.copyto(convert=unique_basename, *local_other.topath()) jret = scopy.copyto(*local_nom.topath()) iret = max(iret, jret) # set remote filename remote_prof.set_filename(scopy.get_remote_filename(fprof)) return iret, remote_prof
def purge_flash(prof, args, **kwargs): """Call --purge_flash action on a server""" # read server informations serv = build_server_from_profile(prof, TYPES.EXEC) iret, output, err = call_generic_service("purge_flash", serv, prof, args) return iret
def local_edit(prof, args, **kwargs): """Call --edit action on a server by using a local editor after copying file if it's remote.""" run = magic.run jobid = prof['jobid'][0] jobname = prof['nomjob'][0] mode = prof['mode'][0] typ = prof['edit_type'][0] to_output = kwargs.get('result_to_output', run['result_to_output']) iret = 0 # read server informations serv = build_server_from_profile(prof, TYPES.COPY_FROM) # flasheur is in the home directory dirname, fname = osp.split( flash_filename("flasheur", jobname, jobid, typ, mode)) run.DBG("file to edit is in %s named %s" % (dirname, fname)) # copy from dirname if not on localhost if not serv.is_localhost(): serv.set_proxy_dir(dirname) dst = osp.join(run['tmp_user'], 'flasheur_%s' % serv.host, fname) is_agla_astout = prof['nomjob'][0].startswith('pre_eda') \ or prof['nomjob'][0].startswith('asrest') if not osp.exists(dst) or is_agla_astout: iret = serv.copyfrom(dst) else: iret = 0 dst = osp.join(get_home_directory(), dirname, fname) if iret == 0 and not to_output: magic.log.info(_(u"edit file %s"), dst) edit_file(run, dst) return iret
def get_export(prof, args, print_output=True, **kwargs): """Call --get_export action on a server.""" # read server informations serv = build_server_from_profile(prof, TYPES.EXEC) vers = kwargs.get('vers', magic.run['aster_vers']) iret, output, err = call_generic_service("get_export", serv, prof, args, {'vers': vers}) if print_output: print3(output) return iret, output
def get_results(prof, args, **kwargs): """Download result files from the server.""" run = magic.run magic.log.info(_(u"get result files from the server")) # read studyid study_prof = get_study_export(prof) if study_prof is None: return 4 studyid = study_prof['studyid'][0] forig = osp.join(run['tmp_user'], "%s.orig.export" % studyid) #peter.zhang, for cygwin forig = forig.replace('\\', '/') # read server informations scopy = build_server_from_profile(study_prof, TYPES.COPY_FROM, jobid=studyid) # get original export iret = scopy.copyfrom(forig) if iret != 0: magic.log.warn(_(u"the results seem already downloaded.")) return iret oprof = AsterProfil(forig, run) magic.run.DBG("original export :\n%s" % repr(oprof), all=True) run_on_localhost = scopy.is_localhost() # copy results files if not run_on_localhost: local_resu = oprof.get_result().get_on_serv(local_full_host) local_nom, local_other = local_resu.get_type('nom', with_completion=True) iret = scopy.copyfrom(convert=unique_basename, *local_other.topath()) jret = scopy.copyfrom(*local_nom.topath()) iret = max(iret, jret) local_resu = local_resu.topath() else: local_resu = [] remote_resu = oprof.get_result().get_on_serv(scopy.host, scopy.user).topath() all = set(oprof.get_result().topath()) all.difference_update(local_resu) all.difference_update(remote_resu) if len(all) > 0: magic.log.warn( _(u"files on a third host should have been copied " "at the end of the calculation (if possible) : %s"), [e.repr() for e in all]) # remove remote repository if iret == 0: scopy.delete_proxy_dir() return iret
def info(prof, args, print_output=True, **kwargs): """Call --info action on a server""" # read server informations serv = build_server_from_profile(prof, TYPES.EXEC) magic.log.info(_(u"retreive configuration informations of the server")) iret, output, err = call_generic_service("info", serv, prof, args) if print_output: print3(output) # already in output magic.run.PrintExitCode = False return iret, output
def get_study_export(prof): """Return the original export.""" run = magic.run # read server informations jobid = prof['jobid'][0] serv = build_server_from_profile(prof, TYPES.COPY_FROM, jobid=jobid) dst = jobid + ".export" iret = serv.copyfrom(dst) if iret != 0: return None # read studyid study_prof = AsterProfil(dst, run) return study_prof
def sendmail(prof, args, **kwargs): """Call --sendmail action on a server. Allow to send a mail even if it's not configured on localhost.""" # put the file on the server run = magic.run num_job = kwargs.get('num_job', run['num_job']) to = kwargs.get('report_to', run['report_to']) jobid = '%s-%s' % (num_job, "sendmail") # read server informations sexec = build_server_from_profile(prof, TYPES.EXEC) scopy = build_server_from_profile(prof, TYPES.COPY_TO, jobid=jobid) # copy text file scopy.copyto(args[1]) rfile = scopy.get_remote_filename(args[1]) targs = ( None, rfile, ) iret, output, err = call_generic_service("sendmail", sexec, prof, targs, {'report_to': to}) scopy.delete_proxy_dir() return iret
def _call_actu(prof, args): """Call --actu action on a server""" jobid = prof['jobid'][0] jobname = prof['nomjob'][0] mode = prof['mode'][0] targs = (None, jobid, jobname, mode) # read server informations serv = build_server_from_profile(prof, TYPES.EXEC) magic.log.info(_(u"ask the server for the job status")) iret, output, err = call_generic_service("actu", serv, prof, targs) magic.log.debug("server returns %s", output) result = parse_actu_result(output) magic.log.info(_(u"job status is %s"), result[0]) return iret, output
def serv_with_reverse_access(prof, args, print_output=True, **kwargs): """The old way to call a server : - the server is called directly through ssh/rsh - reverse access to the client from the server is required to read the export file and other data files. """ action = "serv" # read server informations serv = build_server_from_profile(prof, TYPES.EXEC) filename = get_absolute_path(prof.get_filename()) if not serv.is_localhost(): prof.from_remote_server() prof.WriteExportTo(filename) filename = "%s:%s" % (local_full_host, filename) if local_user != '': filename = "%s@%s" % (local_user, filename) cmd = [ osp.join(serv.get_aster_root(), "bin/", get_plt_exec_name(prof.get_platform(), "as_run")), ] cmd.append("--" + action) cmd.append(filename) # for compatibility with old asrun server cmd.extend(magic.run.get_as_run_args()) iret, output, err = serv.exec_command(cmd, display_forwarding=True) magic.run.DBG( "******************** OUTPUT of as_run --%s ********************" % action, output, "******************** END of as_run --%s ********************" % action, all=True, prefix=" ") res = parse_submission_result(output) if print_output: print3("JOBID=%s QUEUE=%s STUDYID=%s" % res) return iret
def tail(prof, args, print_output=True, **kwargs): """Call --tail action on a server""" jobid = prof['jobid'][0] jobname = prof['nomjob'][0] mode = prof['mode'][0] nbline = prof['tail_nbline'][0] regexp = prof['tail_regexp'][0] targs = (None, jobid, jobname, mode, 'None', nbline, regexp) # read server informations serv = build_server_from_profile(prof, TYPES.EXEC) options = {'result_to_output': True} iret, output, err = call_generic_service("tail", serv, prof, targs, options) if print_output: print3(output) # not expected in output magic.run.PrintExitCode = False return iret, output
def remote_edit(prof, args, **kwargs): """Call --edit action on a server by opening an editor on the server.""" #XXX --edit does not yet support display argument: remote_edit may not work # maybe by adding a sleeping time...? action = "edit" jobid = prof['jobid'][0] jobname = prof['nomjob'][0] mode = prof['mode'][0] typ = prof['edit_type'][0] displ = prof['display'][0] # read server informations serv = build_server_from_profile(prof, TYPES.EXEC) if serv.support_display_forwarding(): # --edit will use os.environ['DISPLAY'] # but this will not work if ssh is run in background... displ = "None:0" # launch the service cmd = [ osp.join(serv.get_aster_root(), "bin/", get_plt_exec_name(prof.get_platform(), "as_run")), ] cmd.append("--" + action) cmd.extend(magic.run.get_rcdir_arg()) targs = (jobid, jobname, mode, typ) cmd.extend(targs) iret, output, err = serv.exec_command(cmd, display_forwarding=True) magic.run.DBG( "******************** OUTPUT of as_run --%s ********************" % action, output, "******************** END of as_run --%s ********************" % action, all=True, prefix=" ") return iret
def get_study_export(prof): """Return the original export.""" run = magic.run # read server informations serv = build_server_from_profile(prof, TYPES.COPY_FROM) jobid = prof['jobid'][0] jobname = prof['nomjob'][0] mode = prof['mode'][0] # flasheur is in the home directory dirname, fname = osp.split( flash_filename("flasheur", jobname, jobid, "export", mode)) run.DBG("export file in %s is named %s" % (dirname, fname)) # copy export file locally serv.set_proxy_dir(dirname) dst = osp.join(run['tmp_user'], 'flasheur_%s' % serv.host, fname) iret = serv.copyfrom(dst) if iret != 0: return None # read studyid study_prof = AsterProfil(dst, run) return study_prof
def serv(prof, args, print_output=True, **kwargs): """Call --serv action on a server.""" run = magic.run #os.system("pause") num_job = kwargs.get('num_job', run['num_job']) # decode special service serv, prof = apply_special_service(prof, run, on_client_side=True) if serv != "": magic.log.info(_(u"special service : %s"), serv) # set studyid (if already defined in prof, use this one) studyid = prof['studyid'][0] if studyid == '': studyid = "%s-%s" % (num_job, prof['mclient'][0].split('.')[0]) # read server informations sexec = build_server_from_profile(prof, TYPES.EXEC) scopy = build_server_from_profile(prof, TYPES.COPY_TO, jobid=studyid) magic.log.info(_(u"prepare execution for %s@%s"), sexec.user, sexec.host) # prepare export for execution with all files present in the remote directory fprof = osp.join(run['tmp_user'], "%s.export" % studyid) forig = osp.join(run['tmp_user'], "%s.orig.export" % studyid) #peter.zhang, for cygwin fprof = fprof.replace('\\', '/') forig = forig.replace('\\', '/') prof.set_filename(forig) prof.WriteExportTo(forig) run.ToDelete(fprof) run.ToDelete(forig) iret, remote_prof = copy_datafiles_on_server(prof, studyid, fprof) if iret != 0: return iret, '' # launch the study cmd = [ osp.join(sexec.get_aster_root(), "bin/", get_plt_exec_name(remote_prof.get_platform(), "as_run")), ] cmd.append("--serv") cmd.append("--num_job=%s" % studyid) cmd.extend(run.get_rcdir_arg()) fname = remote_prof.get_filename() fname = fname.replace('\\', '/') cmd.append(fname) iret, output, err = sexec.exec_command(cmd, display_forwarding=True) run.DBG( "******************** OUTPUT of as_run --serv ********************", output, "******************** ERROR of as_run --serv ********************", err, "******************** END of as_run --serv ********************", all=True, prefix=" ") jobid, queue, stid2 = parse_submission_result(output) run.DBG("The server returns %s and studyid is set to %s" % (stid2, studyid)) if print_output: print3("JOBID=%s QUEUE=%s STUDYID=%s" % (jobid, queue, studyid)) btc = parse_consbtc(output) if btc is not None and print_output: print3("BTCFILE=%s" % btc) if iret != 0: output += os.linesep.join([ output, "******************** ERROR of as_run --serv ********************", err ]) return iret, output