def addOneMani(exchange, logtype, daycode, usepref): SynchUtil.writeManiFile(exchange, logtype, daycode, usepref) manipath = SynchUtil.getManiPath(exchange, logtype, daycode) for line in open(manipath): print line,
def runIntUpdate(exchange, logtype, daycode): SynchUtil.writeManiFile(exchange, logtype, daycode, writesize=True) callHadoop(exchange, logtype, daycode) # delete manifest file manipath = SynchUtil.getManiPath(exchange, logtype, daycode) locRmCall = "rm %s" % (manipath) #print "Local rm call is %s" % (locRmCall) os.system(locRmCall)
def doSimpleSynch(adex, logtype, daycode): print "Running SimpleSynch for adex=%s, logtype=%s, daycode=%s" % ( adex, logtype, daycode) # Generate a manifest file, using appropriate prefix manipath = SynchUtil.writeManiFile(adex, logtype, daycode) hdfsMani = "/tmp/mani/%s" % (manipath) # Put the manifest file on HDFS putCall = "hadoop fs -put %s %s" % (manipath, hdfsMani) print "PutCall: %s" % (hdfsMani) os.system(putCall) # run distcp hdfsDir = "/data/%s/%s/%s/" % (logtype, daycode, adex) distCpCall = "hadoop distcp -f %s %s" % (hdfsMani, hdfsDir) print "DistCpCall: %s" % (distCpCall) os.system(distCpCall) # delete mani file hdfsRmCall = "hadoop fs -rm %s" % (hdfsMani) print hdfsRmCall os.system(hdfsRmCall) # Delete local file locRmCall = "rm %s" % (manipath) print "Local rm call is %s" % (manipath) os.system(locRmCall)
def doSimpleSynch(adex, logtype, daycode): print "Running SimpleSynch for adex=%s, logtype=%s, daycode=%s" % (adex, logtype, daycode) # Generate a manifest file, using appropriate prefix manipath = SynchUtil.writeManiFile(adex, logtype, daycode) hdfsMani = "/tmp/mani/%s" % (manipath) # Put the manifest file on HDFS putCall = "hadoop fs -put %s %s" % (manipath, hdfsMani) print "PutCall: %s" % (hdfsMani) os.system(putCall) # run distcp hdfsDir = "/data/%s/%s/%s/" % (logtype, daycode, adex) distCpCall = "hadoop distcp -f %s %s" % (hdfsMani, hdfsDir) print "DistCpCall: %s" % (distCpCall) os.system(distCpCall) # delete mani file hdfsRmCall = "hadoop fs -rm %s" % (hdfsMani) print hdfsRmCall os.system(hdfsRmCall) # Delete local file locRmCall = "rm %s" % (manipath) print "Local rm call is %s" % (manipath) os.system(locRmCall)