def CopyLogs(self, TopDir, job, dataset, station):
from iceprod.modules import gsiftp
from iceprod.core.dataclasses import I3PostTray
post = I3PostTray()
tx_protocol = "gsiftp://gridftp.icecube.wisc.edu"
if str(station) in ("npx4", "npx3", "npx2-uwa", "NPX"):
tx_protocol = "file:"
FileList = glob.glob(os.getcwd() + '/*out')
FileList.extend(glob.glob(os.getcwd() + '/*log'))
FileList.extend(glob.glob(os.getcwd() + '/*err'))
for f in FileList:
os.system("cp %s %s_%s" % (f, f, str(job)))
Files2cp = glob.glob(os.getcwd() + '/*%s' % str(job))
#change 'skua' to str(station) for production
post.AddModule("gsiftp.URLMultiCopy", "upload_logs")(
("sourcelist", Files2cp),
("destination", "%s/data/sim/scratch/IceSim/SysChkLogs/%s/%s/" %
(tx_protocol, str(station), str(dataset))),
)
post.Execute()
def Execute(self, stats):
if not ipmodule.IPBaseClass.Execute(self, stats): return 0
import icecube.icetray
from iceprod.modules import gsiftp
from iceprod.core.dataclasses import I3PostTray
time.sleep(60)
sys.stdout.flush()
sys.stderr.flush()
post = I3PostTray()
job = int(self.parser.parse("$args(procnum)"))
key = self.parser.parse("$args(key)")
dataset = int(self.parser.parse("$args(dataset)"))
TopDir = expandvars("$TMPTOP")
station = self.parser.parse("$system(gridname)")
#tx_protocol = random.choice(["gsiftp://data1.icecube.wisc.edu","gsiftp://data2.icecube.wisc.edu"])
tx_protocol = "gsiftp://gridftp.icecube.wisc.edu"
if str(station) in ("npx3", "npx2-uwa"): tx_protocol = "file:"
sys.stdout.flush()
sys.stderr.flush()
os.system(""" tar -cf tarred_file.tar %s/*log* %s/*out* %s/*err*""" %
(TopDir, TopDir, TopDir))
post.AddModule("gsiftp.URLCopy", "upload_tar")(
("source",
"file:%s" % os.path.join(os.getcwd(), "tarred_file.tar")),
("destination",
"%s/data/sim/scratch/IceSim/SysChkLogs/%s/%s/%s_tarred_file.tar" %
(tx_protocol, str(station), str(dataset), str(job))),
)
post.Execute()
#raise RuntimeError("table test")
return 0
def Execute(self, stats):
if not ipmodule.IPBaseClass.Execute(self, stats): return 0
import icecube.icetray
from iceprod.modules import gsiftp
from iceprod.core.dataclasses import I3PostTray
post = I3PostTray()
job = int(self.parser.parse("$args(procnum)"))
key = self.parser.parse("$args(key)")
dataset = int(self.parser.parse("$args(dataset)"))
TopDir = expandvars("$TMPTOP")
station = self.parser.parse("$system(gridname)")
tx_protocol = "gsiftp://data1.icecube.wisc.edu"
if str(station) == "npx3": tx_protocol = "file:"
os.system(""" tar -cvf tarred_file.tar %s/*log* %s/*out* %s/*err*""" %
(TopDir, TopDir, TopDir))
post.AddModule("gsiftp.URLCopy", "upload_tar")(
("source",
"file:%s" % os.path.join(os.getcwd(), "tarred_file.tar")),
("destination",
"%s/data/user/ice3simusr/SysChkLogs/%s/%s/%s_%s_tarred_file.tar" %
(tx_protocol, str(station), str(dataset), str(job), str(key))),
)
post.Execute()
logging.debug("*******")
logging.debug(job)
logging.debug(key)
logging.debug("*********")
raise ("!")
return 0
def Execute(self, stats):
if not ipmodule.IPBaseClass.Execute(self, stats): return 0
gcdfile = self.GetParameter('gcdfile')
gcdfile = self.parser.parse(gcdfile) #in case we are using default
outputfile = self.GetParameter('outputfile')
outputfile = self.parser.parse(
outputfile) #in case we are using default
summary = self.GetParameter('summaryfile')
try:
summary = self.parser.parse(summary) #in case we are using default
except:
summary = None
histosfile = self.GetParameter('histosfile')
histosfile = self.parser.parse(
histosfile) #in case we are using default
histosfile_output = self.parser.parse(
'sanityhistos.$args(dataset).$args(procnum).pkl')
target = self.GetParameter('target')
target = self.parser.parse(target) #in case we are using default
if type(target) is list:
import random
target = random.choice(target)
emulate = self.GetParameter('emulate')
basicHisto = self.GetParameter('EnableBasicHistos')
histoPre = self.GetParameter('HistosPrescale')
from iceprod.modules import gsiftp
from iceprod.core.dataclasses import I3PostTray
post = I3PostTray()
job = int(self.parser.parse("$args(procnum)"))
key = self.parser.parse("$args(key)")
dataset = int(self.parser.parse("$args(dataset)"))
moniurl = self.parser.parse("$args(url)")
localcp = boolcast(self.parser.parse("$system(localcp)"))
if summary and os.path.exists(summary):
# Upload summary file to main target without tracking
post.AddModule("gsiftp.URLCopy", "uploadsummary")(
("execute", not job % 10), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), summary)),
("destination", target),
("emulate", emulate),
)
# Upload main physics output file to main target with tracking
post.AddModule("gsiftp.TrackURLCopy", "uploadphysics")(
("execute", True),
("source", "file:%s" % os.path.join(os.getcwd(), outputfile)),
("destination", target),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadphysics2")(
("execute", localcp),
("source", "file:%s" % os.path.join(os.getcwd(), outputfile)),
("destination", self.parser.parse("$system(targeturl)")),
("emulate", emulate),
)
# Upload GCD file to main target with tracking
post.AddModule("gsiftp.TrackURLCopy", "uploadgcd")(
("execute", not job % 1000), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), gcdfile)),
("destination", target),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload GCD file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadgcd2")(
("execute", localcp and not job % 1000), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), gcdfile)),
("destination", self.parser.parse("$system(targeturl)")),
("emulate", emulate),
)
# Upload basichistos file to main target without tracking
post.AddModule("gsiftp.URLCopy", "uploadhisto")(
("execute", basicHisto and histoPre
and not job % histoPre), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), histosfile)),
("destination", os.path.join(target, 'histos', histosfile_output)),
("emulate", emulate),
)
if summary and os.path.exists(summary):
post.SetParser('uploadsummary', self.parser)
post.SetParser('uploadphysics', self.parser)
post.SetParser('uploadphysics2', self.parser)
post.SetParser('uploadgcd', self.parser)
post.SetParser('uploadgcd2', self.parser)
post.SetParser('uploadhisto', self.parser)
# Execute the PostTray
post.Execute()
return 0
def Execute(self, stats):
if not ipmodule.IPBaseClass.Execute(self, stats): return 0
gcdfile = self.GetParameter('gcdfile')
gcdfile = self.parser.parse(gcdfile) #in case we are using default
l2file = self.GetParameter('l2file')
l2file = self.parser.parse(l2file) #in case we are using default
itl2file = self.GetParameter('itl2file')
itl2file = self.parser.parse(itl2file) #in case we are using default
slopfile = self.GetParameter('slopfile')
slopfile = self.parser.parse(slopfile) #in case we are using default
summary = self.GetParameter('summaryfile')
try:
summary = self.parser.parse(summary) #in case we are using default
except:
summary = None
rootfile = self.GetParameter('rootfile')
rootfile = self.parser.parse(rootfile) #in case we are using default
hdf5file = rootfile.replace(".root", ".h5")
itrootfile = self.GetParameter('itrootfile')
itrootfile = self.parser.parse(
itrootfile) #in case we are using default
ithdf5file = itrootfile.replace(".root", ".h5")
histofile = self.GetParameter('histofile')
histofile = self.parser.parse(histofile) #in case we are using default
gapfile = self.GetParameter('gapfile')
gapfile = self.parser.parse(gapfile) #in case we are using default
target2 = self.GetParameter('target2')
target2 = self.parser.parse(target2) #in case we are using default
if type(target2) is list:
import random
target2 = random.choice(target2)
localtarget2 = self.GetParameter('localtarget2')
localtarget2 = self.parser.parse(
localtarget2) #in case we are using default
emulate = self.GetParameter('emulate')
basicHisto = self.GetParameter('EnableBasicHistos')
from iceprod.modules import gsiftp
from iceprod.core.dataclasses import I3PostTray
post = I3PostTray()
dataset = int(self.parser.parse("$args(dataset)"))
localcp = boolcast(self.parser.parse("$system(localcp)"))
job = int(self.parser.parse("$args(procnum)"))
prod = self.GetParameter('production')
itprod = self.GetParameter('IceTopProduction')
uploadlogs = self.GetParameter('uploadlogs')
#local_target2 = self.parser.parse("$system(targeturl)")
local_target2 = localtarget2
local_emulate = emulate
if self.parser.sysopts.has_key('local_emulate'):
local_emulate = boolcast(
self.parser.parse("$system(local_emulate)"))
# production mode?
if prod:
key = self.parser.parse("$args(key)")
moniurl = self.parser.parse("$args(url)")
else:
key = ''
moniurl = ''
local_target2 = target2
localcp = True
local_emulate = emulate
# ---------------------------------------------
if summary and os.path.exists(summary):
# Upload summary file to main target without tracking
post.AddModule("gsiftp.URLCopy", "uploadsummary")(
("execute", False), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), summary)),
("destination", target2),
("emulate", emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
post.AddModule("gsiftp.TrackURLCopy", "uploadl2i3")(
("execute", prod),
("source", "file:%s" % os.path.join(os.getcwd(), l2file)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload SLOP output file to main target with tracking
post.AddModule("gsiftp.TrackURLCopy", "uploadl2slop")(
("execute", prod
and os.path.exists(os.path.join(os.getcwd(), slopfile))),
("source", "file:%s" % os.path.join(os.getcwd(), slopfile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadl2i32")(
("execute", localcp),
("source", "file:%s" % os.path.join(os.getcwd(), l2file)),
("destination", local_target2),
("emulate", local_emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
if itprod:
if prod:
# Decided (CR-WG PhoneCall 23-02-2011) to not send _IT.i3 to data warehouse
# Can be done later if wanted and still available disk space.
post.AddModule("gsiftp.TrackURLCopy", "uploaditl2i3")(
("execute", False),
("source",
"file:%s" % os.path.join(os.getcwd(), itl2file)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploaditl2i32")(
("execute", localcp),
("source", "file:%s" % os.path.join(os.getcwd(), itl2file)),
("destination", local_target2),
("emulate", local_emulate),
)
# ---------------------------------------------
# Upload main physics output root file to main target with tracking
post.AddModule("gsiftp.TrackURLCopy", "uploadl2root")(
("execute", prod
and os.path.exists(os.path.join(os.getcwd(), rootfile))),
("source", "file:%s" % os.path.join(os.getcwd(), rootfile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output hdf5 file to main target with tracking
post.AddModule("gsiftp.TrackURLCopy", "uploadl2hdf5")(
("execute", prod
and os.path.exists(os.path.join(os.getcwd(), hdf5file))),
("source", "file:%s" % os.path.join(os.getcwd(), hdf5file)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadl2root2")(
("execute", localcp
and os.path.exists(os.path.join(os.getcwd(), rootfile))),
("source", "file:%s" % os.path.join(os.getcwd(), rootfile)),
("destination", local_target2),
("emulate", local_emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadl2hdf52")(
("execute", localcp
and os.path.exists(os.path.join(os.getcwd(), hdf5file))),
("source", "file:%s" % os.path.join(os.getcwd(), hdf5file)),
("destination", local_target2),
("emulate", local_emulate),
)
# ---------------------------------------------
# Upload main physics output root file to main target with tracking
if itprod:
if prod:
# Decided (CR-WG PhoneCall 23-02-2011) to not send _IT.root to data warehouse
# Can be done later if wanted and still available disk space.
post.AddModule("gsiftp.TrackURLCopy", "uploaditl2root")(
("execute", False),
("source",
"file:%s" % os.path.join(os.getcwd(), itrootfile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
post.AddModule("gsiftp.TrackURLCopy", "uploaditl2hdf5")(
("execute", False),
("source",
"file:%s" % os.path.join(os.getcwd(), ithdf5file)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploaditl2root2")(
("execute", localcp
and os.path.exists(os.path.join(os.getcwd(), itrootfile))),
("source", "file:%s" % os.path.join(os.getcwd(), itrootfile)),
("destination", local_target2),
("emulate", local_emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploaditl2root2")(
("execute", localcp
and os.path.exists(os.path.join(os.getcwd(), ithdf5file))),
("source", "file:%s" % os.path.join(os.getcwd(), ithdf5file)),
("destination", local_target2),
("emulate", local_emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploadl2histo")(
("execute", prod
and os.path.exists(os.path.join(os.getcwd(), histofile))),
("source", "file:%s" % os.path.join(os.getcwd(), histofile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploadl2gap")(
("execute", prod
and os.path.exists(os.path.join(os.getcwd(), gapfile))),
("source", "file:%s" % os.path.join(os.getcwd(), gapfile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# ---------------------------------------------
# Upload GCD file to main target with tracking
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploadgcd")(
("execute", prod and not job % 1000), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), gcdfile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload GCD file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadgcd2")(
("execute", localcp and not job % 1000), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), gcdfile)),
("destination", local_target2),
("emulate", local_emulate),
)
# ---------------------------------------------
if summary and os.path.exists(summary):
post.SetParser('uploadsummary', self.parser)
if prod:
post.SetParser('uploadl2root', self.parser)
post.SetParser('uploadl2histo', self.parser)
post.SetParser('uploadl2gap', self.parser)
post.SetParser('uploadgcd', self.parser)
post.SetParser('uploadl2i3', self.parser)
if itprod:
post.SetParser('uploaditl2i3', self.parser)
post.SetParser('uploaditl2root', self.parser)
post.SetParser('uploadl2i32', self.parser)
post.SetParser('uploadl2root2', self.parser)
post.SetParser('uploadgcd2', self.parser)
if itprod:
post.SetParser('uploaditl2i32', self.parser)
post.SetParser('uploaditl2root2', self.parser)
if uploadlogs:
stdoutfile = expandvars("$ICEPROD_STDOUT")
stderrfile = expandvars("$ICEPROD_STDERR")
icetraylog = expandvars("$I3_TOPDIR/icetray.%06u.log" % job)
if self.parser.steering.parameters.has_key('CORSIKA::infile'):
corsikalog = self.parser.parse(
"$I3_TOPDIR/$steering(CORSIKA::infile).log")
upload_log(post, corsikalog, os.path.join(target2, "logs/"),
emulate, "", self.parser)
upload_log(post, stdoutfile, os.path.join(target2, "logs/"),
emulate, "", self.parser)
upload_log(post, stderrfile, os.path.join(target2, "logs/"),
emulate, "", self.parser)
upload_log(post, icetraylog, os.path.join(target2, "logs/"),
emulate, "", self.parser)
if localcp:
upload_log(post, stdoutfile,
os.path.join(localtarget2, "logs/"), local_emulate,
"_2", self.parser)
upload_log(post, stderrfile,
os.path.join(localtarget2, "logs/"), local_emulate,
"_2", self.parser)
upload_log(post, icetraylog,
os.path.join(localtarget2, "logs/"), local_emulate,
"_2", self.parser)
upload_log(post, corsikalog,
os.path.join(localtarget2, "logs/"), local_emulate,
"_2", self.parser)
# Execute the PostTray
post.Execute()
return 0
def Execute(self, stats):
if not ipmodule.IPBaseClass.Execute(self, stats): return 0
gcdfile = self.GetParameter('gcdfile')
gcdfile = self.parser.parse(gcdfile) #in case we are using default
l1file = self.GetParameter('l1file')
l1file = self.parser.parse(l1file) #in case we are using default
l2file = self.GetParameter('l2file')
l2file = self.parser.parse(l2file) #in case we are using default
summary = self.GetParameter('summaryfile')
try:
summary = self.parser.parse(summary) #in case we are using default
except:
summary = None
rootfile = self.GetParameter('rootfile')
rootfile = self.parser.parse(rootfile) #in case we are using default
dcorsfile = self.GetParameter('dcorsfile')
dcorsfile = self.parser.parse(dcorsfile) #in case we are using default
histofile = self.GetParameter('histofile')
histofile = self.parser.parse(histofile) #in case we are using default
gapfile = self.GetParameter('gapfile')
gapfile = self.parser.parse(gapfile) #in case we are using default
target1 = self.GetParameter('target1')
target1 = self.parser.parse(target1) #in case we are using default
if type(target1) is list:
import random
target1 = random.choice(target1)
target2 = self.GetParameter('target2')
target2 = self.parser.parse(target2) #in case we are using default
if type(target2) is list:
import random
target2 = random.choice(target2)
target3 = self.GetParameter('target3')
target3 = self.parser.parse(target3) #in case we are using default
if type(target3) is list:
import random
target3 = random.choice(target3)
emulate = self.GetParameter('emulate')
basicHisto = self.GetParameter('EnableBasicHistos')
keepdcors = self.GetParameter('KeepdCorsika')
uploadlogs = self.GetParameter('uploadlogs')
from iceprod.modules import gsiftp
from iceprod.core.dataclasses import I3PostTray
post = I3PostTray()
dataset = int(self.parser.parse("$args(dataset)"))
localcp = boolcast(self.parser.parse("$system(localcp)"))
job = int(self.parser.parse("$args(procnum)"))
prod = self.GetParameter('production')
local_target1 = self.parser.parse("$system(targeturl)")
local_target2 = self.parser.parse("$system(targeturl)")
local_target3 = self.parser.parse("$system(targeturl)")
# production mode?
if prod:
key = self.parser.parse("$args(key)")
moniurl = self.parser.parse("$args(url)")
else:
key = ''
moniurl = ''
local_target1 = target2
local_target2 = target2
localcp = True
# ---------------------------------------------
if summary and os.path.exists(summary):
# Upload summary file to main target without tracking
post.AddModule("gsiftp.URLCopy", "uploadsummary")(
("execute", False), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), summary)),
("destination", target2),
("emulate", emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploadl1ehe")(
("execute", False),
("source", "file:%s" % os.path.join(os.getcwd(), l1file)),
("destination", target1),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadl1ehe2")(
("execute", False),
("source", "file:%s" % os.path.join(os.getcwd(), l1file)),
("destination", local_target1),
("emulate", emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploadl2i3")(
(
"execute", prod
), # BECAUSE THIS DOESN'T WORK, TrackURLCopy will still try to set up a server with empty url
("source", "file:%s" % os.path.join(os.getcwd(), l2file)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadl2i32")(
("execute", localcp),
("source", "file:%s" % os.path.join(os.getcwd(), l2file)),
("destination", local_target2),
("emulate", emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploadl2root")(
("execute", prod),
("source", "file:%s" % os.path.join(os.getcwd(), rootfile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadl2root2")(
("execute", localcp),
("source", "file:%s" % os.path.join(os.getcwd(), rootfile)),
("destination", local_target2),
("emulate", emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
if keepdcors:
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploaddcors")(
("execute", prod),
("source",
"file:%s" % os.path.join(os.getcwd(), dcorsfile)),
("destination", target3),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploaddcors2")(
("execute", localcp),
("source", "file:%s" % os.path.join(os.getcwd(), dcorsfile)),
("destination", local_target3),
("emulate", emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploadl2histo")(
("execute", False),
("source", "file:%s" % os.path.join(os.getcwd(), histofile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadl2histo2")(
("execute", False),
("source", "file:%s" % os.path.join(os.getcwd(), histofile)),
("destination", local_target2),
("emulate", emulate),
)
# ---------------------------------------------
# Upload main physics output file to main target with tracking
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploadl2gap")(
("execute", False),
("source", "file:%s" % os.path.join(os.getcwd(), gapfile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload main physics output file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadl2gap2")(
("execute", False),
("source", "file:%s" % os.path.join(os.getcwd(), gapfile)),
("destination", local_target2),
("emulate", emulate),
)
# ---------------------------------------------
# Upload GCD file to main target with tracking
if prod:
post.AddModule("gsiftp.TrackURLCopy", "uploadgcd")(
("execute", prod and not job % 1000), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), gcdfile)),
("destination", target2),
("monitorURL", moniurl),
("dataset", dataset),
("job", job),
("key", key),
("emulate", emulate),
)
# Upload GCD file to local target without tracking
post.AddModule("gsiftp.URLCopy", "uploadgcd2")(
("execute", localcp and not job % 1000), # prescale
("source", "file:%s" % os.path.join(os.getcwd(), gcdfile)),
("destination", local_target2),
("emulate", emulate),
)
# ---------------------------------------------
if summary and os.path.exists(summary):
post.SetParser('uploadsummary', self.parser)
if prod:
post.SetParser('uploadl1ehe', self.parser)
post.SetParser('uploadl2root', self.parser)
if keepdcors:
post.SetParser('uploaddcors', self.parser)
post.SetParser('uploadl2histo', self.parser)
post.SetParser('uploadl2gap', self.parser)
post.SetParser('uploadgcd', self.parser)
post.SetParser('uploadl2i3', self.parser)
post.SetParser('uploadl1ehe2', self.parser)
post.SetParser('uploadl2i32', self.parser)
post.SetParser('uploadl2root2', self.parser)
if keepdcors:
post.SetParser('uploaddcors2', self.parser)
post.SetParser('uploadl2histo2', self.parser)
post.SetParser('uploadl2gap2', self.parser)
post.SetParser('uploadgcd2', self.parser)
if uploadlogs:
stdoutfile = expandvars("$ICEPROD_STDOUT")
stderrfile = expandvars("$ICEPROD_STDERR")
icetraylog = expandvars("$I3_TOPDIR/icetray.%06u.log" % job)
corsikalog = self.parser.parse(
"$I3_TOPDIR/$steering(CORSIKA::infile).log")
upload_log(post, stdoutfile, os.path.join(target2, "logs/"),
emulate, "", self.parser)
upload_log(post, stderrfile, os.path.join(target2, "logs/"),
emulate, "", self.parser)
upload_log(post, icetraylog, os.path.join(target2, "logs/"),
emulate, "", self.parser)
upload_log(post, corsikalog, os.path.join(target2, "logs/"),
emulate, "", self.parser)
# Execute the PostTray
post.Execute()
return 0
def Execute(self, stats):
if not ipmodule.IPBaseClass.Execute(self, stats): return 0
import icecube.icetray
from iceprod.modules import gsiftp
from iceprod.core.dataclasses import I3PostTray
post = I3PostTray()
job = int(self.parser.parse("$args(procnum)"))
key = self.parser.parse("$args(key)")
dataset = int(self.parser.parse("$args(dataset)"))
TopDir = expandvars("$TMPTOP")
#moniurl = self.parser.parse("$args(url)")
#localcp = boolcast(self.parser.parse("$system(localcp)"))
station = self.parser.parse("$system(gridname)")
tx_protocol = "gsiftp://data1.icecube.wisc.edu"
if str(station) == "npx3": tx_protocol = "file:"
#
#
# os.system("ls -alhtr %s/*log* > log_files"%TopDir)
# post.AddModule("gsiftp.URLCopy","uploadlogfilelist")(
# ("source","file:%s" % os.path.join(os.getcwd(),"log_files")),
# ("destination","%s/data/user/ice3simusr/SysChkLogs/%s/log_files"%(tx_protocol,str(dataset))),
# )
#
# os.system("ls -alhtr %s/*out* > out_files"%TopDir)
# post.AddModule("gsiftp.URLCopy","uploadoutfilelist")(
# ("source","file:%s" % os.path.join(os.getcwd(),"out_files")),
# ("destination","%s/data/user/ice3simusr/SysChkLogs/%s/out_files"%(tx_protocol,str(dataset))),
# )
#
# os.system("ls -alhtr %s/*err* > error_files"%TopDir)
# post.AddModule("gsiftp.URLCopy","uploaderrfilelist")(
# ("source","file:%s" % os.path.join(os.getcwd(),"error_files")),
# ("destination","%s/data/user/ice3simusr/SysChkLogs/%s/error_files"%(tx_protocol,str(dataset))),
# #("destination","file:/data/user/ice3simusr/SysChkLogs/%s/error_files"%str(dataset)),
# )
#
# log_files = glob.glob(TopDir+"/*log*")
# for log_file in log_files:
# mod_name = "upload_" + os.path.basename(log_file)
# post.AddModule("gsiftp.URLCopy",mod_name)(
# ("source","file:%s" % os.path.join(os.getcwd(),log_file)),
# ("destination","%s/data/user/ice3simusr/SysChkLogs/%s/%s"%(tx_protocol,str(dataset),os.path.basename(log_file))),
# )
#
# out_files = glob.glob(TopDir+"/*out*")
# for out_file in out_files:
# mod_name = "upload_" + os.path.basename(out_file)
# post.AddModule("gsiftp.URLCopy",mod_name)(
# ("source","file:%s" % os.path.join(os.getcwd(),out_file)),
# ("destination","%s/data/user/ice3simusr/SysChkLogs/%s/%s"%(tx_protocol,str(dataset),os.path.basename(out_file))),
# )
#
# err_files = glob.glob(TopDir+"/*err*")
# for err_file in err_files:
# mod_name = "upload_" + os.path.basename(err_file)
# post.AddModule("gsiftp.URLCopy",mod_name)(
# ("source","file:%s" % os.path.join(os.getcwd(),err_file)),
# ("destination","%s/data/user/ice3simusr/SysChkLogs/%s/%s"%(tx_protocol,str(dataset),os.path.basename(err_file))),
# #("destination","file:/data/user/ice3simusr/SysChkLogs/%s/%s"%(str(dataset),os.path.basename(err_file))),
# )
os.system(""" tar -cvf tarred_file.tar %s/*log* %s/*out* %s/*err*""" %
(TopDir, TopDir, TopDir))
post.AddModule("gsiftp.URLCopy", "upload_tar")(
("source",
"file:%s" % os.path.join(os.getcwd(), "tarred_file.tar")),
("destination",
"%s/data/user/ice3simusr/SysChkLogs/%s/%s/%s_%s_tarred_file.tar" %
(tx_protocol, str(station), str(dataset), str(job), str(key))),
)
post.Execute()
print("*******")
print(job)
print(key)
#print """ tar -cvf tarred_file.tar %s/*log* %s/*out* %s/*err*"""%(TopDir,TopDir,TopDir)
print("*********")
raise ("!")
return 0