def addOneMani(exchange, logtype, daycode, usepref):
SynchUtil.writeManiFile(exchange, logtype, daycode, usepref)
manipath = SynchUtil.getManiPath(exchange, logtype, daycode)
for line in open(manipath):
print line,
def addOneMani(exchange, logtype, daycode, usepref): SynchUtil.writeManiFile(exchange, logtype, daycode, usepref) manipath = SynchUtil.getManiPath(exchange, logtype, daycode) for line in open(manipath): print line,
def runIntUpdate(exchange, logtype, daycode): SynchUtil.writeManiFile(exchange, logtype, daycode, writesize=True) callHadoop(exchange, logtype, daycode) # delete manifest file manipath = SynchUtil.getManiPath(exchange, logtype, daycode) locRmCall = "rm %s" % (manipath) #print "Local rm call is %s" % (locRmCall) os.system(locRmCall)
def runIntUpdate(exchange, logtype, daycode):
SynchUtil.writeManiFile(exchange, logtype, daycode, writesize=True)
callHadoop(exchange, logtype, daycode)
# delete manifest file
manipath = SynchUtil.getManiPath(exchange, logtype, daycode)
locRmCall = "rm %s" % (manipath)
#print "Local rm call is %s" % (locRmCall)
os.system(locRmCall)
def doSimpleSynch(adex, logtype, daycode):
print "Running SimpleSynch for adex=%s, logtype=%s, daycode=%s" % (
adex, logtype, daycode)
# Generate a manifest file, using appropriate prefix
manipath = SynchUtil.writeManiFile(adex, logtype, daycode)
hdfsMani = "/tmp/mani/%s" % (manipath)
# Put the manifest file on HDFS
putCall = "hadoop fs -put %s %s" % (manipath, hdfsMani)
print "PutCall: %s" % (hdfsMani)
os.system(putCall)
# run distcp
hdfsDir = "/data/%s/%s/%s/" % (logtype, daycode, adex)
distCpCall = "hadoop distcp -f %s %s" % (hdfsMani, hdfsDir)
print "DistCpCall: %s" % (distCpCall)
os.system(distCpCall)
# delete mani file
hdfsRmCall = "hadoop fs -rm %s" % (hdfsMani)
print hdfsRmCall
os.system(hdfsRmCall)
# Delete local file
locRmCall = "rm %s" % (manipath)
print "Local rm call is %s" % (manipath)
os.system(locRmCall)
def doSimpleSynch(adex, logtype, daycode): print "Running SimpleSynch for adex=%s, logtype=%s, daycode=%s" % (adex, logtype, daycode) # Generate a manifest file, using appropriate prefix manipath = SynchUtil.writeManiFile(adex, logtype, daycode) hdfsMani = "/tmp/mani/%s" % (manipath) # Put the manifest file on HDFS putCall = "hadoop fs -put %s %s" % (manipath, hdfsMani) print "PutCall: %s" % (hdfsMani) os.system(putCall) # run distcp hdfsDir = "/data/%s/%s/%s/" % (logtype, daycode, adex) distCpCall = "hadoop distcp -f %s %s" % (hdfsMani, hdfsDir) print "DistCpCall: %s" % (distCpCall) os.system(distCpCall) # delete mani file hdfsRmCall = "hadoop fs -rm %s" % (hdfsMani) print hdfsRmCall os.system(hdfsRmCall) # Delete local file locRmCall = "rm %s" % (manipath) print "Local rm call is %s" % (manipath) os.system(locRmCall)
