# Delete local file
locRmCall = "rm %s" % (manipath)
print "Local rm call is %s" % (manipath)
os.system(locRmCall)
if __name__ == "__main__":
if not len(sys.argv) == 4:
print "Usage: SimpleSynch <all|adex> <big|mini|comp|logtype> <yest|daycode>"
sys.exit(1)
exclist = SynchUtil.getCheckExcList(sys.argv[1])
loglist = SynchUtil.getCheckLogList(sys.argv[2])
daylist = SynchUtil.getCheckDayList(sys.argv[3])
# need to make sure we're running in a folder where the .mani files
# can be written
os.chdir('/var/log/cronlogs/hdfs/manifiles/')
for onex in exclist:
for logtype in loglist:
for daycode in daylist:
if not SynchUtil.nfsFilesExist(onex, logtype, daycode):
print "No NFS files for %s %s %s" % (onex, logtype,
daycode)
continue
#os.system("echo \"calling simplesynch for %s %s %s\" | wall" % (onex, logtype, daycode))
# TODO: change this to use LocalConf
hadcall = "hadoop jar /local/bin/jars/adnetik.jar %s %s %s" % (dmclass, daycode, tempfile)
print "\nHadoop call is : \n\t%s" % (hadcall)
os.system(hadcall)
#print "\nFinished with %s" % (dmclass)
if __name__ == "__main__":
if not len(sys.argv) == 2:
print "Usage: BmUpdate <yest|daycode>"
sys.exit(1)
daylist = SynchUtil.getCheckDayList(sys.argv[1])
daycode = daylist[0]
# This is kind of the hacky way to do things
sys.path.append("/local/src/python/util")
import SimpleMail
logmail = SimpleMail.SimpleMail("DbStageSlice")
javalist = []
# TODO: going to roll all of this into a single Java file StagingInfoManager, obviate this Python script
#javalist.append("com.adnetik.data_management.Special2Staging")
javalist.append("com.adnetik.userindex.StagingInfoManager")
#javalist.append("com.adnetik.data_management.Click2Staging")
#javalist.append("com.adnetik.data_management.Negative2Staging")
# need to make sure we're running in a folder where we have write permissions,
# otherwise we won't be able to write the manifest file
os.chdir('/mnt/src/cronjobs/')
exclist = []
daylist = []
loglist = []
for line in sys.stdin:
if len(line.strip().split('\t')) < 3:
continue
(excCode, logType, dayCode) = line.strip().split('\t')
#print "Syncing %s %s %s" % (excCode, logType, dayCode)
exclist.append(SynchUtil.getCheckExcList(excCode)[0])
loglist.append(SynchUtil.getCheckLogList(logType)[0])
daylist.append(SynchUtil.getCheckDayList(dayCode)[0])
idxlist = []
for i in range(len(exclist)):
print "Syncing %s %s %s" % (exclist[i], loglist[i], daylist[i])
ConcatLzoSynch.runLogSync(exclist[i], loglist[i], daylist[i])
idxlist.add(SynchUtil.getHdfsPath(exclist[i], loglist[i], daylist[i]))
for toidx in idxlist:
ConcatLzoSynch.runIndexer(toidx)
exclist = []
daylist = []
loglist = []
for line in sys.stdin:
if len(line.strip().split('\t')) < 3:
continue
(excCode, logType, dayCode) = line.strip().split('\t')
#print "Syncing %s %s %s" % (excCode, logType, dayCode)
exclist.append(SynchUtil.getCheckExcList(excCode)[0])
loglist.append(SynchUtil.getCheckLogList(logType)[0])
daylist.append(SynchUtil.getCheckDayList(dayCode)[0])
idxlist = []
for i in range(len(exclist)):
print "Syncing %s %s %s" % (exclist[i], loglist[i], daylist[i])
ConcatLzoSynch.runLogSync(exclist[i], loglist[i], daylist[i])
idxlist.add(SynchUtil.getHdfsPath(exclist[i], loglist[i], daylist[i]))
for toidx in idxlist:
ConcatLzoSynch.runIndexer(toidx)
