print hdfsRmCall
os.system(hdfsRmCall)
# Delete local file
locRmCall = "rm %s" % (manipath)
print "Local rm call is %s" % (manipath)
os.system(locRmCall)
if __name__ == "__main__":
if not len(sys.argv) == 4:
print "Usage: SimpleSynch <all|adex> <big|mini|comp|logtype> <yest|daycode>"
sys.exit(1)
exclist = SynchUtil.getCheckExcList(sys.argv[1])
loglist = SynchUtil.getCheckLogList(sys.argv[2])
daylist = SynchUtil.getCheckDayList(sys.argv[3])
# need to make sure we're running in a folder where the .mani files
# can be written
os.chdir('/var/log/cronlogs/hdfs/manifiles/')
for onex in exclist:
for logtype in loglist:
for daycode in daylist:
if not SynchUtil.nfsFilesExist(onex, logtype, daycode):
print "No NFS files for %s %s %s" % (onex, logtype,
daycode)
continue
# need to make sure we're running in a folder where we have write permissions,
# otherwise we won't be able to write the manifest file
os.chdir('/mnt/src/cronjobs/')
exclist = []
daylist = []
loglist = []
for line in sys.stdin:
if len(line.strip().split('\t')) < 3:
continue
(excCode, logType, dayCode) = line.strip().split('\t')
#print "Syncing %s %s %s" % (excCode, logType, dayCode)
exclist.append(SynchUtil.getCheckExcList(excCode)[0])
loglist.append(SynchUtil.getCheckLogList(logType)[0])
daylist.append(SynchUtil.getCheckDayList(dayCode)[0])
idxlist = []
for i in range(len(exclist)):
print "Syncing %s %s %s" % (exclist[i], loglist[i], daylist[i])
ConcatLzoSynch.runLogSync(exclist[i], loglist[i], daylist[i])
idxlist.add(SynchUtil.getHdfsPath(exclist[i], loglist[i], daylist[i]))
for toidx in idxlist:
ConcatLzoSynch.runIndexer(toidx)
def runTrackUpdate(daycode):
hadoopsys = "hadoop jar %s %s %s" % (SynchUtil.JAR_PATH,
UPDATE_TRACK_CLASS, daycode)
print "Hadoop call is %s" % (hadoopsys)
os.system(hadoopsys)
if __name__ == "__main__":
"""
This is a one-time operation to copy the impression logs and update
the tracking file
"""
exclist = SynchUtil.getCheckExcList('all')
daylist = SynchUtil.getCheckDayList(sys.argv[1])
logtype = 'imp'
for daycode in daylist:
for onex in exclist:
print "Uploading logs for %s %s %s" % (onex, logtype, daycode)
ConcatLzoSynch.runLogSync(onex, logtype, daycode)
ConcatLzoSynch.runIndexer()
# Now we have indexed LZO files, so we can run Java UpdateTrackFile
runTrackUpdate(daycode)
os.chdir('/mnt/src/cronjobs/')
exclist = []
daylist = []
loglist = []
for line in sys.stdin:
if len(line.strip().split('\t')) < 3:
continue
(excCode, logType, dayCode) = line.strip().split('\t')
#print "Syncing %s %s %s" % (excCode, logType, dayCode)
exclist.append(SynchUtil.getCheckExcList(excCode)[0])
loglist.append(SynchUtil.getCheckLogList(logType)[0])
daylist.append(SynchUtil.getCheckDayList(dayCode)[0])
idxlist = []
for i in range(len(exclist)):
print "Syncing %s %s %s" % (exclist[i], loglist[i], daylist[i])
ConcatLzoSynch.runLogSync(exclist[i], loglist[i], daylist[i])
idxlist.add(SynchUtil.getHdfsPath(exclist[i], loglist[i], daylist[i]))
for toidx in idxlist:
ConcatLzoSynch.runIndexer(toidx)
# delete manifest file
manipath = SynchUtil.getManiPath(exchange, logtype, daycode)
locRmCall = "rm %s" % (manipath)
#print "Local rm call is %s" % (locRmCall)
os.system(locRmCall)
if __name__ == "__main__":
if not len(sys.argv) == 4:
print "Usage: InterestUpdateWrapper <all|adex> <big|mini|comp|logtype> <yest|daycode|filename>"
sys.exit(1)
# need to make sure we're running in a folder where we have write permissions,
# otherwise we won't be able to write the manifest file
os.chdir('/mnt/src/cronjobs/')
exclist = SynchUtil.getCheckExcList(sys.argv[1])
loglist = SynchUtil.getCheckLogList(sys.argv[2])
daylist = SynchUtil.getCheckDayList(sys.argv[3])
for onex in exclist:
for logtype in loglist:
for daycode in daylist:
runIntUpdate(onex, logtype, daycode)
def runTrackUpdate(daycode):
hadoopsys = "hadoop jar %s %s %s" % (SynchUtil.JAR_PATH, UPDATE_TRACK_CLASS, daycode)
print "Hadoop call is %s" % ( hadoopsys )
os.system(hadoopsys)
if __name__ == "__main__":
"""
This is a one-time operation to copy the impression logs and update
the tracking file
"""
exclist = SynchUtil.getCheckExcList('all')
daylist = SynchUtil.getCheckDayList(sys.argv[1])
logtype = 'imp'
for daycode in daylist:
for onex in exclist:
print "Uploading logs for %s %s %s" % (onex, logtype, daycode)
ConcatLzoSynch.runLogSync(onex, logtype, daycode)
ConcatLzoSynch.runIndexer()
# Now we have indexed LZO files, so we can run Java UpdateTrackFile
runTrackUpdate(daycode)