os.system(hdfsRmCall)
# Delete local file
locRmCall = "rm %s" % (manipath)
print "Local rm call is %s" % (manipath)
os.system(locRmCall)
if __name__ == "__main__":
if not len(sys.argv) == 4:
print "Usage: SimpleSynch <all|adex> <big|mini|comp|logtype> <yest|daycode>"
sys.exit(1)
exclist = SynchUtil.getCheckExcList(sys.argv[1])
loglist = SynchUtil.getCheckLogList(sys.argv[2])
daylist = SynchUtil.getCheckDayList(sys.argv[3])
# need to make sure we're running in a folder where the .mani files
# can be written
os.chdir('/var/log/cronlogs/hdfs/manifiles/')
for onex in exclist:
for logtype in loglist:
for daycode in daylist:
if not SynchUtil.nfsFilesExist(onex, logtype, daycode):
print "No NFS files for %s %s %s" % (onex, logtype,
daycode)
continue
exclist = []
daylist = []
loglist = []
for line in sys.stdin:
if len(line.strip().split('\t')) < 3:
continue
(excCode, logType, dayCode) = line.strip().split('\t')
#print "Syncing %s %s %s" % (excCode, logType, dayCode)
exclist.append(SynchUtil.getCheckExcList(excCode)[0])
loglist.append(SynchUtil.getCheckLogList(logType)[0])
daylist.append(SynchUtil.getCheckDayList(dayCode)[0])
idxlist = []
for i in range(len(exclist)):
print "Syncing %s %s %s" % (exclist[i], loglist[i], daylist[i])
ConcatLzoSynch.runLogSync(exclist[i], loglist[i], daylist[i])
idxlist.add(SynchUtil.getHdfsPath(exclist[i], loglist[i], daylist[i]))
for toidx in idxlist:
ConcatLzoSynch.runIndexer(toidx)
# need to make sure we're running in a folder where we have write permissions,
# otherwise we won't be able to write the manifest file
os.chdir('/mnt/src/cronjobs/')
exclist = []
daylist = []
loglist = []
for line in sys.stdin:
if len(line.strip().split('\t')) < 3:
continue
(excCode, logType, dayCode) = line.strip().split('\t')
#print "Syncing %s %s %s" % (excCode, logType, dayCode)
exclist.append(SynchUtil.getCheckExcList(excCode)[0])
loglist.append(SynchUtil.getCheckLogList(logType)[0])
daylist.append(SynchUtil.getCheckDayList(dayCode)[0])
idxlist = []
for i in range(len(exclist)):
print "Syncing %s %s %s" % (exclist[i], loglist[i], daylist[i])
ConcatLzoSynch.runLogSync(exclist[i], loglist[i], daylist[i])
idxlist.add(SynchUtil.getHdfsPath(exclist[i], loglist[i], daylist[i]))
for toidx in idxlist:
ConcatLzoSynch.runIndexer(toidx)
# delete manifest file
manipath = SynchUtil.getManiPath(exchange, logtype, daycode)
locRmCall = "rm %s" % (manipath)
#print "Local rm call is %s" % (locRmCall)
os.system(locRmCall)
if __name__ == "__main__":
if not len(sys.argv) == 4:
print "Usage: InterestUpdateWrapper <all|adex> <big|mini|comp|logtype> <yest|daycode|filename>"
sys.exit(1)
# need to make sure we're running in a folder where we have write permissions,
# otherwise we won't be able to write the manifest file
os.chdir('/mnt/src/cronjobs/')
exclist = SynchUtil.getCheckExcList(sys.argv[1])
loglist = SynchUtil.getCheckLogList(sys.argv[2])
daylist = SynchUtil.getCheckDayList(sys.argv[3])
for onex in exclist:
for logtype in loglist:
for daycode in daylist:
runIntUpdate(onex, logtype, daycode)