def assignHosts(theBoxId):
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
freehosts = getFreeHost()
if (freehosts):
pendingTasks = dbconnSync.execute(
"select * from tasks where theBoxId = '" +
str(theBoxId).rstrip().lstrip() + "' and status = " +
str(constants.ouiSync_tasks_status_pending) +
" order by priority desc",
dictionary=True)
if (not isinstance(pendingTasks, int)):
for x in pendingTasks:
if (x):
try:
dbconnSync.execute(
"update tasks set status = " +
str(constants.ouiSync_tasks_status_assigned) +
" , hostId = '" + str(freehosts['id']) +
"' where theBoxId = '" + str(x['theBoxId']) + "'")
dbconnSync.execute(
"update hosts set cpuFree = cpuFree-1 where id = '"
+ str(freehosts['id']) + "'")
except:
print(str(sys.exc_info()))
return (0)
return (1)
def update():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid , ip, totalCpus = getLocalHostDetails()
loads = loadAvg()
try:
dbconnSync.execute("insert into hosts (id,ip,cpuTotal,cpuFree,isAlive) value ('"+ str(hostid).rstrip().lstrip() +"','"+ str(ip).rstrip().lstrip() +"','"+ str(totalCpus).rstrip().lstrip() +"','"+ str(totalCpus).rstrip().lstrip() +"','"+ str(constants.ouiSync_hosts_isAlive_online) +"') \
on duplicate key update ip='"+ str(ip).rstrip().lstrip() +"', cpuTotal='"+ str(totalCpus) +"', cpuFree='"+ str(totalCpus) +"', isAlive='"+ str(constants.ouiSync_hosts_isAlive_online) +"'")
dbconnSync.execute("update hosts set load1='"+ str(loads[0]).rstrip().lstrip() +"', load2 = '"+ str(loads[1]).rstrip().lstrip() +"', load3 = '"+ str(loads[2]).rstrip().lstrip() +"' where id='"+ str(hostid) +"'")
except:
print(str(sys.exc_info()))
def getHostsByLoad():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
freehosts = {}
rawHosts = dbconnSync.execute("select * from hosts where enabled = "+ str(constants.ouiSync_hosts_enabled_enabled) \
+" and isAlive = "+ str(constants.ouiSync_hosts_isAlive_online) \
+" and load1 < cpuTotal order by weight desc,load1 desc",dictionary=True)
if(not isinstance(rawHosts, int)):
for x in rawHosts:
if(x):
return(x)
return(0)
def getHostsByLoad():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
freehosts = {}
rawHosts = dbconnSync.execute("select * from hosts where enabled = "+ str(constants.ouiSync_hosts_enabled_enabled) \
+" and isAlive = "+ str(constants.ouiSync_hosts_isAlive_online) \
+" and load1 < cpuTotal order by weight desc,load1 desc",dictionary=True)
if (not isinstance(rawHosts, int)):
for x in rawHosts:
if (x):
return (x)
return (0)
def doSync(syncDict):
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid , ip, totalCpus = getLocalHostDetails()
try:
dbconnSync.execute("update tasks set status = "+ str(constants.ouiSync_tasks_status_running) +" where hostid = '"+ str(hostid) +"' and theBoxId = '"+ str(syncDict['theBoxId']) +"'")
except:
print(str(sys.exc_info()))
return(0)
rawBoxes = dbconnDevices.execute("select * from theBox where id='"+ str(syncDict['theBoxId']) +"'",dictionary=True)
if(not isinstance(rawBoxes,int)):
thebox = rawBoxes[-1]
rsynccmd = constants.rsync +" \""+ syncDict['path'] +"\" "+ str(constants.theBoxUserName) +"@"+ thebox['ip'] +":"+ syncDict['destinationPath']
try:
os.chdir(syncDict['path'])
out = subprocess.Popen(rsynccmd,shell=True)
except:
print(str(sys.exc_info()))
while(True):
boxAlive = checkClient(thebox["ip"])
if(boxAlive == 0):
try:
out.terminate()
except:
pass
exitcode = out.poll()
if(exitcode != None):
if(exitcode == 0):
try:
dbconnSync.execute("update tasks set status = "+ str(constants.ouiSync_tasks_status_done) +" where theBoxId = '"+ str(syncDict['theBoxId']) +"'")
except:
print(str(sys.exc_info()))
else:
try:
dbconnSync.execute("update tasks set status = "+ str(constants.ouiSync_tasks_status_pending) +" where theBoxId = '"+ str(syncDict['theBoxId']) +"'")
except:
print(str(sys.exc_info()))
try:
dbconnDevices.execute("update theBox set isAlive = "+ str(constants.ouiDevices_theBox_isAlive_offline) +" where id = '"+ str(thebox['id']) +"'")
except:
print(str(sys.exc_info()))
break
time.sleep(5)
try:
dbconnSync.execute("update hosts set cpuFree = cpuFree+1 where id = '"+ str(hostid) +"'")
except:
print(str(sys.exc_info()))
return(0)
def doSyncProcess():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid , ip, totalCpus = getLocalHostDetails()
procpool = Pool(processes=int(totalCpus))
jobs = []
while (1):
try:
tasksAssigned = dbconnSync.execute("select * from tasks where status = "+ str(constants.ouiSync_tasks_status_assigned) +" and hostid = '"+ str(hostid) +"'",dictionary=True)
if(not isinstance(tasksAssigned,int)):
for x in tasksAssigned:
proc = procpool.apply_async(func=doSync,args=(x,))
jobs.append(proc)
except:
print(str(sys.exc_info()))
time.sleep(1)
def assignHosts(theBoxId):
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
freehosts = getFreeHost()
if(freehosts):
pendingTasks = dbconnSync.execute("select * from tasks where theBoxId = '"+ str(theBoxId).rstrip().lstrip() +"' and status = "+ str(constants.ouiSync_tasks_status_pending) +" order by priority desc",dictionary=True)
if(not isinstance(pendingTasks, int)):
for x in pendingTasks:
if(x):
try:
dbconnSync.execute("update tasks set status = "+ str(constants.ouiSync_tasks_status_assigned) +" , hostId = '"+ str(freehosts['id']) +"' where theBoxId = '"+ str(x['theBoxId']) +"'")
dbconnSync.execute("update hosts set cpuFree = cpuFree-1 where id = '"+ str(freehosts['id']) +"'")
except:
print(str(sys.exc_info()))
return(0)
return(1)
def doSyncProcess():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid, ip, totalCpus = getLocalHostDetails()
procpool = Pool(processes=int(totalCpus))
jobs = []
while (1):
try:
tasksAssigned = dbconnSync.execute(
"select * from tasks where status = " +
str(constants.ouiSync_tasks_status_assigned) +
" and hostid = '" + str(hostid) + "'",
dictionary=True)
if (not isinstance(tasksAssigned, int)):
for x in tasksAssigned:
proc = procpool.apply_async(func=doSync, args=(x, ))
jobs.append(proc)
except:
print(str(sys.exc_info()))
time.sleep(1)
def update():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid, ip, totalCpus = getLocalHostDetails()
loads = loadAvg()
try:
dbconnSync.execute(
"insert into hosts (id,ip,cpuTotal,cpuFree,isAlive) value ('" +
str(hostid).rstrip().lstrip() + "','" + str(ip).rstrip().lstrip() +
"','" + str(totalCpus).rstrip().lstrip() + "','" +
str(totalCpus).rstrip().lstrip() + "','" +
str(constants.ouiSync_hosts_isAlive_online) + "') \
on duplicate key update ip='" +
str(ip).rstrip().lstrip() + "', cpuTotal='" + str(totalCpus) +
"', cpuFree='" + str(totalCpus) + "', isAlive='" +
str(constants.ouiSync_hosts_isAlive_online) + "'")
dbconnSync.execute("update hosts set load1='" +
str(loads[0]).rstrip().lstrip() + "', load2 = '" +
str(loads[1]).rstrip().lstrip() + "', load3 = '" +
str(loads[2]).rstrip().lstrip() + "' where id='" +
str(hostid) + "'")
except:
print(str(sys.exc_info()))
# print("lib : "+ libDir)
import dbOuiSync
import constants
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--boxid", dest='boxid', help='boxid to sync')
parser.add_argument("-l",
"--list",
dest='islist',
action='store_true',
help='list all syncs')
args = parser.parse_args()
dbconnSync = dbOuiSync.db()
rawSyncs = dbconnSync.execute("select * from tasks", dictionary=True)
if (args.islist):
if (not isinstance(rawSyncs, int)):
for x in rawSyncs:
print(str(x['theBoxId']) + " : " + str(x['path']))
elif (args.boxid):
try:
print("updating the resync")
dbconnSync.execute("update tasks set status = " +
str(constants.ouiSync_tasks_status_pending) +
" where theBoxId = '" +
str(args.boxid).lstrip().rstrip() + "'")
except:
print(sys.exc_info())
import dbOuiDevices
import dbOuiSync
import sha512sum
parser = argparse.ArgumentParser()
parser.add_argument("-i","--boxid",dest='boxid',help='boxid to sync')
parser.add_argument("-p","--path",dest='path',help='root path to sync')
parser.add_argument("-d","--destinationpath",dest='destpath',help='destination root path to sync')
parser.add_argument("-l","--list",dest='islist',action='store_true',help='list all syncs')
parser.add_argument("-b","--listBox",dest='islistbox',action='store_true',help='list all box ids')
args = parser.parse_args()
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
rawBoxes = dbconnDevices.execute("select * from theBox",dictionary=True)
rawSyncs = dbconnSync.execute("select * from tasks",dictionary=True)
def getFiles(path,destpath,theBoxId):
rootpath = os.path.abspath(path)
a = os.walk(path)
for root,dirs,files in a:
for b in files:
bchecksum = str(sha512sum.checksum(os.path.join(os.path.abspath(root),b))).rstrip().lstrip()
bpath = str(os.path.join(root,b)).replace(rootpath, "").lstrip(".").lstrip(os.sep).rstrip().lstrip()
print(bchecksum +":"+ rootpath +":"+ bpath)
try:
dbconnSync.execute("insert into taskJobs (theBoxId,checksum,path,destinationpath,file) value ('"+ str(theBoxId).rstrip().lstrip() +"','"+ str(bchecksum).rstrip().lstrip() +"','"+ str(rootpath).rstrip().lstrip() +"','"+ str(destpath).rstrip().lstrip() +"','"+ str(bpath).rstrip().lstrip() +"')")
except:
print(str(sys.exc_info()))
def doSync(syncDict):
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid, ip, totalCpus = getLocalHostDetails()
try:
dbconnSync.execute("update tasks set status = " +
str(constants.ouiSync_tasks_status_running) +
" where hostid = '" + str(hostid) +
"' and theBoxId = '" + str(syncDict['theBoxId']) +
"'")
except:
print(str(sys.exc_info()))
return (0)
rawBoxes = dbconnDevices.execute("select * from theBox where id='" +
str(syncDict['theBoxId']) + "'",
dictionary=True)
if (not isinstance(rawBoxes, int)):
thebox = rawBoxes[-1]
rsynccmd = constants.rsync + " \"" + syncDict['path'] + "\" " + str(
constants.theBoxUserName
) + "@" + thebox['ip'] + ":" + syncDict['destinationPath']
try:
os.chdir(syncDict['path'])
out = subprocess.Popen(rsynccmd, shell=True)
except:
print(str(sys.exc_info()))
while (True):
boxAlive = checkClient(thebox["ip"])
if (boxAlive == 0):
try:
out.terminate()
except:
pass
exitcode = out.poll()
if (exitcode != None):
if (exitcode == 0):
try:
dbconnSync.execute(
"update tasks set status = " +
str(constants.ouiSync_tasks_status_done) +
" where theBoxId = '" + str(syncDict['theBoxId']) +
"'")
except:
print(str(sys.exc_info()))
else:
try:
dbconnSync.execute(
"update tasks set status = " +
str(constants.ouiSync_tasks_status_pending) +
" where theBoxId = '" + str(syncDict['theBoxId']) +
"'")
except:
print(str(sys.exc_info()))
try:
dbconnDevices.execute(
"update theBox set isAlive = " +
str(constants.ouiDevices_theBox_isAlive_offline) +
" where id = '" + str(thebox['id']) + "'")
except:
print(str(sys.exc_info()))
break
time.sleep(5)
try:
dbconnSync.execute(
"update hosts set cpuFree = cpuFree+1 where id = '" +
str(hostid) + "'")
except:
print(str(sys.exc_info()))
return (0)