def updateTheBox(self, headers):
dbconn = dbOuiDevices.db()
try:
dbconn.execute("insert into theBox (id,ip,isAlive) values \
('" + str(headers['id']).rstrip().lstrip() + "','" +
str(headers['ip']).rstrip().lstrip() + "','" +
str(1) + "') \
on duplicate key update ip='" + str(headers['ip']).rstrip().lstrip() +
"', isAlive='" + str(1) + "'")
except:
print(str(sys.exc_info()))
return (0)
try:
print("/usr/bin/ssh-keyscan " +
str(headers['ip']).rstrip().lstrip() +
" >> ~/.ssh/known_hosts")
os.system("/usr/bin/ssh-keyscan " +
str(headers['ip']).rstrip().lstrip() +
" >> ~/.ssh/known_hosts")
print(constants.rsync + " " + constants.theBoxUserName + "@" +
str(headers['ip']).rstrip().lstrip() + ":" +
constants.theBoxUserSave + " " +
constants.theBackendRootUsers)
os.system(constants.rsync + " " + constants.theBoxUserName + "@" +
str(headers['ip']).rstrip().lstrip() + ":" +
constants.theBoxUserSave + " " +
constants.theBackendRootUsers)
except:
print(str(sys.exc_info()))
return (0)
syncServer.assignHosts(headers['id'])
return (1)
def assignHosts(theBoxId):
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
freehosts = getFreeHost()
if (freehosts):
pendingTasks = dbconnSync.execute(
"select * from tasks where theBoxId = '" +
str(theBoxId).rstrip().lstrip() + "' and status = " +
str(constants.ouiSync_tasks_status_pending) +
" order by priority desc",
dictionary=True)
if (not isinstance(pendingTasks, int)):
for x in pendingTasks:
if (x):
try:
dbconnSync.execute(
"update tasks set status = " +
str(constants.ouiSync_tasks_status_assigned) +
" , hostId = '" + str(freehosts['id']) +
"' where theBoxId = '" + str(x['theBoxId']) + "'")
dbconnSync.execute(
"update hosts set cpuFree = cpuFree-1 where id = '"
+ str(freehosts['id']) + "'")
except:
print(str(sys.exc_info()))
return (0)
return (1)
def update():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid , ip, totalCpus = getLocalHostDetails()
loads = loadAvg()
try:
dbconnSync.execute("insert into hosts (id,ip,cpuTotal,cpuFree,isAlive) value ('"+ str(hostid).rstrip().lstrip() +"','"+ str(ip).rstrip().lstrip() +"','"+ str(totalCpus).rstrip().lstrip() +"','"+ str(totalCpus).rstrip().lstrip() +"','"+ str(constants.ouiSync_hosts_isAlive_online) +"') \
on duplicate key update ip='"+ str(ip).rstrip().lstrip() +"', cpuTotal='"+ str(totalCpus) +"', cpuFree='"+ str(totalCpus) +"', isAlive='"+ str(constants.ouiSync_hosts_isAlive_online) +"'")
dbconnSync.execute("update hosts set load1='"+ str(loads[0]).rstrip().lstrip() +"', load2 = '"+ str(loads[1]).rstrip().lstrip() +"', load3 = '"+ str(loads[2]).rstrip().lstrip() +"' where id='"+ str(hostid) +"'")
except:
print(str(sys.exc_info()))
def getHostsByLoad():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
freehosts = {}
rawHosts = dbconnSync.execute("select * from hosts where enabled = "+ str(constants.ouiSync_hosts_enabled_enabled) \
+" and isAlive = "+ str(constants.ouiSync_hosts_isAlive_online) \
+" and load1 < cpuTotal order by weight desc,load1 desc",dictionary=True)
if(not isinstance(rawHosts, int)):
for x in rawHosts:
if(x):
return(x)
return(0)
def getHostsByLoad():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
freehosts = {}
rawHosts = dbconnSync.execute("select * from hosts where enabled = "+ str(constants.ouiSync_hosts_enabled_enabled) \
+" and isAlive = "+ str(constants.ouiSync_hosts_isAlive_online) \
+" and load1 < cpuTotal order by weight desc,load1 desc",dictionary=True)
if (not isinstance(rawHosts, int)):
for x in rawHosts:
if (x):
return (x)
return (0)
def doSync(syncDict):
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid , ip, totalCpus = getLocalHostDetails()
try:
dbconnSync.execute("update tasks set status = "+ str(constants.ouiSync_tasks_status_running) +" where hostid = '"+ str(hostid) +"' and theBoxId = '"+ str(syncDict['theBoxId']) +"'")
except:
print(str(sys.exc_info()))
return(0)
rawBoxes = dbconnDevices.execute("select * from theBox where id='"+ str(syncDict['theBoxId']) +"'",dictionary=True)
if(not isinstance(rawBoxes,int)):
thebox = rawBoxes[-1]
rsynccmd = constants.rsync +" \""+ syncDict['path'] +"\" "+ str(constants.theBoxUserName) +"@"+ thebox['ip'] +":"+ syncDict['destinationPath']
try:
os.chdir(syncDict['path'])
out = subprocess.Popen(rsynccmd,shell=True)
except:
print(str(sys.exc_info()))
while(True):
boxAlive = checkClient(thebox["ip"])
if(boxAlive == 0):
try:
out.terminate()
except:
pass
exitcode = out.poll()
if(exitcode != None):
if(exitcode == 0):
try:
dbconnSync.execute("update tasks set status = "+ str(constants.ouiSync_tasks_status_done) +" where theBoxId = '"+ str(syncDict['theBoxId']) +"'")
except:
print(str(sys.exc_info()))
else:
try:
dbconnSync.execute("update tasks set status = "+ str(constants.ouiSync_tasks_status_pending) +" where theBoxId = '"+ str(syncDict['theBoxId']) +"'")
except:
print(str(sys.exc_info()))
try:
dbconnDevices.execute("update theBox set isAlive = "+ str(constants.ouiDevices_theBox_isAlive_offline) +" where id = '"+ str(thebox['id']) +"'")
except:
print(str(sys.exc_info()))
break
time.sleep(5)
try:
dbconnSync.execute("update hosts set cpuFree = cpuFree+1 where id = '"+ str(hostid) +"'")
except:
print(str(sys.exc_info()))
return(0)
def doSyncProcess():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid , ip, totalCpus = getLocalHostDetails()
procpool = Pool(processes=int(totalCpus))
jobs = []
while (1):
try:
tasksAssigned = dbconnSync.execute("select * from tasks where status = "+ str(constants.ouiSync_tasks_status_assigned) +" and hostid = '"+ str(hostid) +"'",dictionary=True)
if(not isinstance(tasksAssigned,int)):
for x in tasksAssigned:
proc = procpool.apply_async(func=doSync,args=(x,))
jobs.append(proc)
except:
print(str(sys.exc_info()))
time.sleep(1)
def assignHosts(theBoxId):
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
freehosts = getFreeHost()
if(freehosts):
pendingTasks = dbconnSync.execute("select * from tasks where theBoxId = '"+ str(theBoxId).rstrip().lstrip() +"' and status = "+ str(constants.ouiSync_tasks_status_pending) +" order by priority desc",dictionary=True)
if(not isinstance(pendingTasks, int)):
for x in pendingTasks:
if(x):
try:
dbconnSync.execute("update tasks set status = "+ str(constants.ouiSync_tasks_status_assigned) +" , hostId = '"+ str(freehosts['id']) +"' where theBoxId = '"+ str(x['theBoxId']) +"'")
dbconnSync.execute("update hosts set cpuFree = cpuFree-1 where id = '"+ str(freehosts['id']) +"'")
except:
print(str(sys.exc_info()))
return(0)
return(1)
def doSyncProcess():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid, ip, totalCpus = getLocalHostDetails()
procpool = Pool(processes=int(totalCpus))
jobs = []
while (1):
try:
tasksAssigned = dbconnSync.execute(
"select * from tasks where status = " +
str(constants.ouiSync_tasks_status_assigned) +
" and hostid = '" + str(hostid) + "'",
dictionary=True)
if (not isinstance(tasksAssigned, int)):
for x in tasksAssigned:
proc = procpool.apply_async(func=doSync, args=(x, ))
jobs.append(proc)
except:
print(str(sys.exc_info()))
time.sleep(1)
def updateTheBox(self,headers):
dbconn = dbOuiDevices.db()
try:
dbconn.execute("insert into theBox (id,ip,isAlive) values \
('"+ str(headers['id']).rstrip().lstrip() +"','"+ str(headers['ip']).rstrip().lstrip() +"','"+ str(1) +"') \
on duplicate key update ip='"+ str(headers['ip']).rstrip().lstrip() +"', isAlive='"+ str(1) +"'")
except:
print(str(sys.exc_info()))
return(0)
try:
print("/usr/bin/ssh-keyscan "+ str(headers['ip']).rstrip().lstrip() +" >> ~/.ssh/known_hosts")
os.system("/usr/bin/ssh-keyscan "+ str(headers['ip']).rstrip().lstrip() +" >> ~/.ssh/known_hosts")
print(constants.rsync +" "+ constants.theBoxUserName +"@"+ str(headers['ip']).rstrip().lstrip() +":"+ constants.theBoxUserSave +" "+ constants.theBackendRootUsers)
os.system(constants.rsync +" "+ constants.theBoxUserName +"@"+ str(headers['ip']).rstrip().lstrip() +":"+ constants.theBoxUserSave +" "+ constants.theBackendRootUsers)
except:
print(str(sys.exc_info()))
return(0)
syncServer.assignHosts(headers['id'])
return(1)
def update():
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid, ip, totalCpus = getLocalHostDetails()
loads = loadAvg()
try:
dbconnSync.execute(
"insert into hosts (id,ip,cpuTotal,cpuFree,isAlive) value ('" +
str(hostid).rstrip().lstrip() + "','" + str(ip).rstrip().lstrip() +
"','" + str(totalCpus).rstrip().lstrip() + "','" +
str(totalCpus).rstrip().lstrip() + "','" +
str(constants.ouiSync_hosts_isAlive_online) + "') \
on duplicate key update ip='" +
str(ip).rstrip().lstrip() + "', cpuTotal='" + str(totalCpus) +
"', cpuFree='" + str(totalCpus) + "', isAlive='" +
str(constants.ouiSync_hosts_isAlive_online) + "'")
dbconnSync.execute("update hosts set load1='" +
str(loads[0]).rstrip().lstrip() + "', load2 = '" +
str(loads[1]).rstrip().lstrip() + "', load3 = '" +
str(loads[2]).rstrip().lstrip() + "' where id='" +
str(hostid) + "'")
except:
print(str(sys.exc_info()))
"-s",
"--states",
dest='states',
help='comma seperated states to add.\nEg:karnataka, maharastra, telangana')
parser.add_argument("-c",
"--country",
dest='country',
help='country to which the states belong.\nEg:india')
parser.add_argument("-l",
"--list",
dest='islist',
action='store_true',
help='list all the states')
args = parser.parse_args()
dbconn = dbOuiDevices.db()
if (args.islist):
#print("listing all the id types for devices")
raw = dbconn.execute("select * from states", dictionary=True)
print(raw)
if (not isinstance(raw, int)):
for x in raw:
print(x['country'] + ":" + x['state'])
else:
if (args.states):
print("states : " + str(args.states))
if (args.country):
print("country : " + str(args.country))
for x in args.states.split(","):
dirSelf = os.path.dirname(os.path.realpath(__file__))
libDir = dirSelf.rstrip(os.sep).rstrip("toolsDevice").rstrip(os.sep).rstrip("bin").rstrip(os.sep) + os.sep + "lib"
sys.path.append(libDir)
# print("lib : "+ libDir)
import dbOuiDevices
parser = argparse.ArgumentParser()
parser.add_argument("-s","--states",dest='states',help='comma seperated states to add.\nEg:karnataka, maharastra, telangana')
parser.add_argument("-c","--country",dest='country',help='country to which the states belong.\nEg:india')
parser.add_argument("-l","--list",dest='islist',action='store_true',help='list all the states')
args = parser.parse_args()
dbconn = dbOuiDevices.db()
if(args.islist):
#print("listing all the id types for devices")
raw = dbconn.execute("select * from states",dictionary=True)
print(raw)
if(not isinstance(raw,int)):
for x in raw:
print(x['country'] +":"+ x['state'])
else:
if(args.states):
print("states : "+ str(args.states))
if(args.country):
print("country : "+ str(args.country))
for x in args.states.split(","):
import dbOuiDevices
import dbOuiSync
import sha512sum
parser = argparse.ArgumentParser()
parser.add_argument("-i","--boxid",dest='boxid',help='boxid to sync')
parser.add_argument("-p","--path",dest='path',help='root path to sync')
parser.add_argument("-d","--destinationpath",dest='destpath',help='destination root path to sync')
parser.add_argument("-l","--list",dest='islist',action='store_true',help='list all syncs')
parser.add_argument("-b","--listBox",dest='islistbox',action='store_true',help='list all box ids')
args = parser.parse_args()
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
rawBoxes = dbconnDevices.execute("select * from theBox",dictionary=True)
rawSyncs = dbconnSync.execute("select * from tasks",dictionary=True)
def getFiles(path,destpath,theBoxId):
rootpath = os.path.abspath(path)
a = os.walk(path)
for root,dirs,files in a:
for b in files:
bchecksum = str(sha512sum.checksum(os.path.join(os.path.abspath(root),b))).rstrip().lstrip()
bpath = str(os.path.join(root,b)).replace(rootpath, "").lstrip(".").lstrip(os.sep).rstrip().lstrip()
print(bchecksum +":"+ rootpath +":"+ bpath)
try:
dbconnSync.execute("insert into taskJobs (theBoxId,checksum,path,destinationpath,file) value ('"+ str(theBoxId).rstrip().lstrip() +"','"+ str(bchecksum).rstrip().lstrip() +"','"+ str(rootpath).rstrip().lstrip() +"','"+ str(destpath).rstrip().lstrip() +"','"+ str(bpath).rstrip().lstrip() +"')")
except:
"--destinationpath",
dest='destpath',
help='destination root path to sync')
parser.add_argument("-l",
"--list",
dest='islist',
action='store_true',
help='list all syncs')
parser.add_argument("-b",
"--listBox",
dest='islistbox',
action='store_true',
help='list all box ids')
args = parser.parse_args()
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
rawBoxes = dbconnDevices.execute("select * from theBox", dictionary=True)
rawSyncs = dbconnSync.execute("select * from tasks", dictionary=True)
def getFiles(path, destpath, theBoxId):
rootpath = os.path.abspath(path)
a = os.walk(path)
for root, dirs, files in a:
for b in files:
bchecksum = str(
sha512sum.checksum(os.path.join(os.path.abspath(root),
b))).rstrip().lstrip()
bpath = str(os.path.join(root, b)).replace(
rootpath, "").lstrip(".").lstrip(os.sep).rstrip().lstrip()
def doSync(syncDict):
dbconnDevices = dbOuiDevices.db()
dbconnSync = dbOuiSync.db()
hostid, ip, totalCpus = getLocalHostDetails()
try:
dbconnSync.execute("update tasks set status = " +
str(constants.ouiSync_tasks_status_running) +
" where hostid = '" + str(hostid) +
"' and theBoxId = '" + str(syncDict['theBoxId']) +
"'")
except:
print(str(sys.exc_info()))
return (0)
rawBoxes = dbconnDevices.execute("select * from theBox where id='" +
str(syncDict['theBoxId']) + "'",
dictionary=True)
if (not isinstance(rawBoxes, int)):
thebox = rawBoxes[-1]
rsynccmd = constants.rsync + " \"" + syncDict['path'] + "\" " + str(
constants.theBoxUserName
) + "@" + thebox['ip'] + ":" + syncDict['destinationPath']
try:
os.chdir(syncDict['path'])
out = subprocess.Popen(rsynccmd, shell=True)
except:
print(str(sys.exc_info()))
while (True):
boxAlive = checkClient(thebox["ip"])
if (boxAlive == 0):
try:
out.terminate()
except:
pass
exitcode = out.poll()
if (exitcode != None):
if (exitcode == 0):
try:
dbconnSync.execute(
"update tasks set status = " +
str(constants.ouiSync_tasks_status_done) +
" where theBoxId = '" + str(syncDict['theBoxId']) +
"'")
except:
print(str(sys.exc_info()))
else:
try:
dbconnSync.execute(
"update tasks set status = " +
str(constants.ouiSync_tasks_status_pending) +
" where theBoxId = '" + str(syncDict['theBoxId']) +
"'")
except:
print(str(sys.exc_info()))
try:
dbconnDevices.execute(
"update theBox set isAlive = " +
str(constants.ouiDevices_theBox_isAlive_offline) +
" where id = '" + str(thebox['id']) + "'")
except:
print(str(sys.exc_info()))
break
time.sleep(5)
try:
dbconnSync.execute(
"update hosts set cpuFree = cpuFree+1 where id = '" +
str(hostid) + "'")
except:
print(str(sys.exc_info()))
return (0)