def pcsErrorHandler(self, e):
"""pcs error handler"""
emsg = e.read()
logging.error('[%s]: HTTP error=>%d, result=>%s', self.getName(), e.code, emsg)
edata = json.loads(emsg)
error = {'http': e.code, 'code': edata['error_code']}
self.errorHandler(error)
def send(self, msg):
"""send message to related msg Queue"""
q = self.findQ(msg.header.rUid)
if q:
q.put(msg)
else:
logging.error('[UMsgBus]: send: find rUid %d queue failure', msg.header.rUid)
def refreshToken(self):
"""Refresh access token"""
super(BaiduCloudAPI, self).refreshToken()
api_url = self.cf.get("BaiduCloud", "openapi_url") + "/token"
logging.debug('[BaiduPlugin]: refreshToken: URL=>%s', api_url)
param = {
"grant_type": "refresh_token",
"refresh_token": self.cf.get("BaiduCloud", "refresh_token"),
"client_id": self.cf.get("BaiduCloud", "api_key"),
"client_secret": self.cf.get("BaiduCloud", "secret_key")
}
try:
ret_fp = cloud_post(api_url, param)
data = json.load(ret_fp)
logging.debug(
'[BaiduPlugin]: getToken: access_token=>%s, refresh_token=>%s',
data["access_token"], data["refresh_token"])
self.cf.set("BaiduCloud", "access_token", data["access_token"])
self.cf.set("BaiduCloud", "refresh_token", data["refresh_token"])
self.cf.set("BaiduCloud", "session_key", data["session_key"])
self.cf.set("BaiduCloud", "session_secret", data["session_secret"])
self.cf.set("BaiduCloud", "scope", data["scope"])
ret_fp.close()
except ValueError, e:
logging.error('[%s]: refreshToken JSON load error %s', e)
def loadAllPlugins(self):
"""load all plugins from destant path"""
p_paths = ConfManager.getManager().getPluginPaths()
pluginList = ConfManager.getManager().getValue('common', 'plugins')
for p in p_paths:
logging.debug('[%s]: loadAllPlugins from %s', 'PluginManager', p)
#TODO: __import__ the module into our script
try:
dirs = os.listdir(p)
for d in dirs:
tmp = '%s%s%s' % (p, os.sep, d)
if os.path.isdir(tmp):
module_name = 'UniFileSync.plugins.%s' % d
module_path = '%s%s%sPlugin.py' % (tmp, os.sep, d)
imp.load_source('', module_path)
pl = {'name': d, 'path': module_path}
if pl not in pluginList:
pluginList.append(pl)
except OSError as exc:
logging.error('loadAllPlugins listdir error %d', OSError.errno)
ConfManager.getManager().setValue('common', 'plugins', pluginList)
logging.debug('[%s]: loadAllPlugins and save %s into configuration', 'PluginManager', pluginList)
ConfManager.getManager().save()
def run(self):
"""UServer entry"""
super(UServer, self).run()
#TODO: below should be set by ConfManager
if self.isEnableSocket:
self.sock.bind(('localhost', 8089))
self.sock.listen(5)
while self.isRunning:
try:
conn, addr = self.sock.accept()
try:
conn.settimeout(5)
buf = conn.recv(1024) #TODO: should be also in ConfManager
req = json.loads(buf)
logging.debug('[UniFileSync]: action %s, param %s', req['action'], req['param'])
#TODO: make it common for function usage
res, data = self.getHandler(req['action'])(req['param'])
ret = {'action': req['action'], 'param': {'data': data}, 'res': res, 'type': 'ack'}
conn.send(json.dumps(ret))
except socket.timeout:
logging.info('[UniFileSync]: socket time out from %s', addr)
except KeyError, e:
logging.error('[%s]: Key Error with param %s', self.getName(), req['param'])
finally:
conn.close()
except KeyboardInterrupt:
print 'Press Ctrl+C'
self.stop()
def getSyncFolder(self, path):
"""get sync folder from path"""
dirpath = path
try:
index = path.rindex(os.sep) + 1
dirpath = path[index:]
except ValueError, KeyError:
logging.error('[%s]: getSyncFolder failure with no so seperator', self.getName())
def send(self, msg):
"""send message to related msg Queue"""
q = self.findQ(msg.header.rUid)
if q:
q.put(msg)
else:
logging.error('[UMsgBus]: send: find rUid %d queue failure',
msg.header.rUid)
def pcsErrorHandler(self, e):
"""pcs error handler"""
emsg = e.read()
logging.error('[%s]: HTTP error=>%d, result=>%s', self.getName(),
e.code, emsg)
edata = json.loads(emsg)
error = {'http': e.code, 'code': edata['error_code']}
self.errorHandler(error)
def regUniID(self, msgUniID):
"""register new msg uni ID"""
if msgUniID not in self.__uni_ID_list:
MsgBus.__lock.acquire()
self.__uni_ID_list.append(msgUniID)
MsgBus.__lock.release()
return E_OK
logging.error('regUniID failure due to Unique Message ID %d already registered', msgUniID)
return E_INVILD_PARAM
def regQ(self, msgUniID, msgQueue):
"""register msg Queue"""
if msgUniID not in self.__uni_ID_list:
logging.error('regQ failure due to no Unique Message ID registered')
return E_INVILD_PARAM
UMsgBus.__lock.acquire()
self.__queue_table[msgUniID] = msgQueue
UMsgBus.__lock.release()
return E_OK
def broadcast(self, msg):
"""broadcast message to all listeners"""
sUid = msg.header.sUid
for l in self.getListeners(sUid):
q = self.findQ(l)
if q:
q.put(msg)
else:
logging.error('[UMsgBus]: broadcast: find sUid %d listener %d queue failure', msg.header.sUid, l)
def regQ(self, msgUniID, msgQueue):
"""register msg Queue"""
if msgUniID not in self.__uni_ID_list:
logging.error(
'regQ failure due to no Unique Message ID registered')
return E_INVILD_PARAM
UMsgBus.__lock.acquire()
self.__queue_table[msgUniID] = msgQueue
UMsgBus.__lock.release()
return E_OK
def register_openers():
"""register some openers into urlib2"""
#Enable media post, proxy, cookie
#logging.debug('%s', __handlers)
res = E_OK
try:
urllib2.install_opener(urllib2.build_opener(*__handlers))
except Exception, e:
logging.error('[register_openers]: exception occurs %s', e)
res = E_OPEN_FAIL
def msgLoop(self):
"""message loop for processing message"""
while self.__isRunning:
try:
msg = self.__msgQueue.get(True)
if msg and msg.header.mtype:
self.__operTable[msg.header.mtype](msg)
except Queue.Empty, e:
logging.error('[%s]: msgLoop timeout in %d with empty item', self.getName(), timeout)
finally:
def broadcast(self, msg):
"""broadcast message to all listeners"""
sUid = msg.header.sUid
for l in self.getListeners(sUid):
q = self.findQ(l)
if q:
q.put(msg)
else:
logging.error(
'[UMsgBus]: broadcast: find sUid %d listener %d queue failure',
msg.header.sUid, l)
def regUniID(self, msgUniID):
"""register new msg uni ID"""
if msgUniID not in self.__uni_ID_list:
MsgBus.__lock.acquire()
self.__uni_ID_list.append(msgUniID)
MsgBus.__lock.release()
return E_OK
logging.error(
'regUniID failure due to Unique Message ID %d already registered',
msgUniID)
return E_INVILD_PARAM
def getMsg(self, timeout=0):
"""process message within timeout"""
msg = None
try:
msg = self.__msgQueue.get(True, timeout)
except Queue.Empty:
logging.error('[%s]: getMsg timeout in %d with empty item', self.getName(), timeout)
finally:
pass
return msg
def msgLoop(self):
"""message loop for processing message"""
while self.__isRunning:
try:
msg = self.__msgQueue.get(True)
if msg and msg.header.mtype:
self.__operTable[msg.header.mtype](msg)
except Queue.Empty, e:
logging.error('[%s]: msgLoop timeout in %d with empty item',
self.getName(), timeout)
finally:
def getMsg(self, timeout=0):
"""process message within timeout"""
msg = None
try:
msg = self.__msgQueue.get(True, timeout)
except Queue.Empty:
logging.error('[%s]: getMsg timeout in %d with empty item',
self.getName(), timeout)
finally:
pass
return msg
def startHandler(self, param):
"""start handler for actors"""
logging.debug('[%s]: startHandler with parm %s', self.getName(), param)
if param['name']:
name = param['name'].lower()
else:
name = ''
res = E_OK
watchpaths = ConfManager.getManager().getValue('common', 'folders')
if name == 'all' or name == '':
if self.cActor.isRunning or self.fsMonitor.isRunning:
return E_ACTOR_ALIVE, None
self.cActor.start()
self.__startActors.append(self.cActor)
self.fsMonitor.start()
self.__startActors.append(self.fsMonitor)
time.sleep(2)
for w in watchpaths:
self.watchHandler({'path': w, 'action': 'add'}, False)
#logging.debug('[%s]: add watch path %s from configuration file', self.getName(), w)
elif name == 'monitor':
if self.fsMonitor.isRunning:
return E_ACTOR_ALIVE, None
self.fsMonitor.start()
self.__startActors.append(self.fsMonitor)
time.sleep(2)
for w in watchpaths:
self.watchHandler({'path': w, 'action': 'add'})
logging.debug(
'[%s]: add watch path %s from configuration file',
self.getName(), w)
elif name == 'cloud':
if self.cActor.isRunning:
return E_ACTOR_ALIVE, None
self.cActor.start()
self.__startActors.append(self.cActor)
else:
logging.error('[%s]: startHandler with name %s error',
self.getName(), param['name'])
res = E_INVILD_PARAM
return res, None
def getCloudInfo(self):
url = self.cf.get("BaiduCloud", "pcs_url") + "/quota"
param = {"method": "info", "access_token": self.cf.get("BaiduCloud", "access_token")}
res = E_API_ERR
data = "getCloudInfo failure"
try:
f = cloud_get(url, param)
data = json.load(f)
res = E_OK
f.close()
except ValueError, e:
logging.error('[%s]: getCloudInfo JSON load error %s', e)
res = E_VALUE_ERR
def stopHandler(self, param):
"""stop handler for actors"""
logging.debug('[%s]: stopHandler with parm %s', self.getName(), param)
if param['name']:
name = param['name'].lower()
else:
name = ''
res = E_ACTOR_DEAD
if name == 'all' or name == '':
#The msg is broadcase
msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, None, True)
self.notifyListeners(msg)
for a in self.__startActors:
rmsg = self.getMsg(2)
if rmsg:
res = rmsg.body['result']
logging.info('[UniFileSync]: stop server...')
self.stop()
elif name == 'monitor':
if not self.fsMonitor.isRunning:
res = E_ACTOR_DEAD
msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP,
MSG_UNIQUE_ID_T_FS_MONITOR, True)
self.msgBus.send(msg)
rmsg = self.getMsg(2)
if rmsg:
res = rmsg.body['result']
elif name == 'cloud':
if not self.cActor.isRunning:
res = E_ACTOR_DEAD
msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP,
MSG_UNIQUE_ID_T_CLOUD_ACTOR, True)
self.msgBus.send(msg)
rmsg = self.getMsg(2)
if rmsg:
res = rmsg.body['result']
else:
logging.error('[%s]: stopHandler with name %s error',
self.getName(), param['name'])
res = E_INVILD_PARAM
return res, None
def startHandler(self, param):
"""start handler for actors"""
logging.debug('[%s]: startHandler with parm %s', self.getName(), param);
if param['name']:
name = param['name'].lower()
else:
name = ''
res = E_OK
watchpaths = ConfManager.getManager().getValue('common', 'folders')
if name == 'all' or name == '':
if self.cActor.isRunning or self.fsMonitor.isRunning:
return E_ACTOR_ALIVE, None
self.cActor.start()
self.__startActors.append(self.cActor)
self.fsMonitor.start()
self.__startActors.append(self.fsMonitor)
time.sleep(2)
for w in watchpaths:
self.watchHandler({'path': w, 'action': 'add'}, False)
#logging.debug('[%s]: add watch path %s from configuration file', self.getName(), w)
elif name == 'monitor':
if self.fsMonitor.isRunning:
return E_ACTOR_ALIVE, None
self.fsMonitor.start()
self.__startActors.append(self.fsMonitor)
time.sleep(2)
for w in watchpaths:
self.watchHandler({'path': w, 'action': 'add'})
logging.debug('[%s]: add watch path %s from configuration file', self.getName(), w)
elif name == 'cloud':
if self.cActor.isRunning:
return E_ACTOR_ALIVE, None
self.cActor.start()
self.__startActors.append(self.cActor)
else:
logging.error('[%s]: startHandler with name %s error', self.getName(), param['name'])
res = E_INVILD_PARAM
return res, None
def getCloudInfo(self):
url = self.cf.get("BaiduCloud", "pcs_url") + "/quota"
param = {
"method": "info",
"access_token": self.cf.get("BaiduCloud", "access_token")
}
res = E_API_ERR
data = "getCloudInfo failure"
try:
f = cloud_get(url, param)
data = json.load(f)
res = E_OK
f.close()
except ValueError, e:
logging.error('[%s]: getCloudInfo JSON load error %s', e)
res = E_VALUE_ERR
def stopHandler(self, param):
"""stop handler for actors"""
logging.debug('[%s]: stopHandler with parm %s', self.getName(), param);
if param['name']:
name = param['name'].lower()
else:
name = ''
res = E_ACTOR_DEAD
if name == 'all' or name == '':
#The msg is broadcase
msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, None, True)
self.notifyListeners(msg)
for a in self.__startActors:
rmsg = self.getMsg(2)
if rmsg:
res = rmsg.body['result']
logging.info('[UniFileSync]: stop server...')
self.stop()
elif name == 'monitor':
if not self.fsMonitor.isRunning:
res = E_ACTOR_DEAD
msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, MSG_UNIQUE_ID_T_FS_MONITOR, True)
self.msgBus.send(msg)
rmsg = self.getMsg(2)
if rmsg:
res = rmsg.body['result']
elif name == 'cloud':
if not self.cActor.isRunning:
res = E_ACTOR_DEAD
msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, MSG_UNIQUE_ID_T_CLOUD_ACTOR, True)
self.msgBus.send(msg)
rmsg = self.getMsg(2)
if rmsg:
res = rmsg.body['result']
else:
logging.error('[%s]: stopHandler with name %s error', self.getName(), param['name'])
res = E_INVILD_PARAM
return res, None
def run(self):
"""UServer entry"""
super(UServer, self).run()
#TODO: below should be set by ConfManager
if self.isEnableSocket:
self.sock.bind(('localhost', 8089))
self.sock.listen(5)
while self.isRunning:
try:
conn, addr = self.sock.accept()
try:
conn.settimeout(5)
buf = conn.recv(
1024) #TODO: should be also in ConfManager
req = json.loads(buf)
logging.debug('[UniFileSync]: action %s, param %s',
req['action'], req['param'])
#TODO: make it common for function usage
res, data = self.getHandler(req['action'])(
req['param'])
ret = {
'action': req['action'],
'param': {
'data': data
},
'res': res,
'type': 'ack'
}
conn.send(json.dumps(ret))
except socket.timeout:
logging.info('[UniFileSync]: socket time out from %s',
addr)
except KeyError, e:
logging.error('[%s]: Key Error with param %s',
self.getName(), req['param'])
finally:
conn.close()
except KeyboardInterrupt:
print 'Press Ctrl+C'
self.stop()
def msgLoop(self):
"""message loop for server"""
res = E_OK
data = None
while self.isRunning:
try:
msg = self.msgQueue.get(True)
if msg and msg.header.mtype:
#This is msg sent by FSMonitor and get result from CloudActor
if 'result' in msg.body:
for c in self.__callbackList:
c(msg.body)
continue
res, data = self.getHandler(msg.header.mtype)(msg.body)
if msg.header.ack:
for c in self.__callbackList:
c({'result': res, 'data': data})
except Queue.Empty, e:
logging.error('[%s]: msgLoop timeout with empty item', self.getName())
finally:
def msgLoop(self):
"""message loop for server"""
res = E_OK
data = None
while self.isRunning:
try:
msg = self.msgQueue.get(True)
if msg and msg.header.mtype:
#This is msg sent by FSMonitor and get result from CloudActor
if 'result' in msg.body:
for c in self.__callbackList:
c(msg.body)
continue
res, data = self.getHandler(msg.header.mtype)(msg.body)
if msg.header.ack:
for c in self.__callbackList:
c({'result': res, 'data': data})
except Queue.Empty, e:
logging.error('[%s]: msgLoop timeout with empty item',
self.getName())
finally:
def refreshToken(self):
"""Refresh access token"""
super(BaiduCloudAPI, self).refreshToken()
api_url = self.cf.get("BaiduCloud", "openapi_url") + "/token"
logging.debug('[BaiduPlugin]: refreshToken: URL=>%s', api_url)
param = {"grant_type": "refresh_token", "refresh_token": self.cf.get("BaiduCloud", "refresh_token"),
"client_id": self.cf.get("BaiduCloud", "api_key"),
"client_secret": self.cf.get("BaiduCloud", "secret_key")}
try:
ret_fp = cloud_post(api_url, param)
data = json.load(ret_fp)
logging.debug('[BaiduPlugin]: getToken: access_token=>%s, refresh_token=>%s', data["access_token"], data["refresh_token"])
self.cf.set("BaiduCloud", "access_token", data["access_token"])
self.cf.set("BaiduCloud", "refresh_token", data["refresh_token"])
self.cf.set("BaiduCloud", "session_key", data["session_key"])
self.cf.set("BaiduCloud", "session_secret", data["session_secret"])
self.cf.set("BaiduCloud", "scope", data["scope"])
ret_fp.close()
except ValueError, e:
logging.error('[%s]: refreshToken JSON load error %s', e)
def getAPI(self):
"""get plugin cloud API"""
if not self.__API:
logging.error(self.name + ' does not have any cloud API')
return self.__API
def getAPI(self):
"""get plugin cloud API"""
if not self.__API:
logging.error(self.name + ' does not have any cloud API')
return self.__API
param['path'] = self.cf.get("BaiduCloud", "app_path") + "/" + syncPath
res = E_API_ERR
try:
fp = open(filePath)
ret_fp = cloud_multi_post(self.cf.get("BaiduCloud", "upload_url"),
param, {'file': fp})
res = json.load(ret_fp)
ret_fp.close()
fp.close()
except urllib2.HTTPError, e:
self.pcsErrorHandler(e)
res = E_API_ERR
except ValueError, e:
logging.error('[%s]: uploadSingleFile JSON load error %s', e)
res = E_VALUE_ERR
return self.parseResult(res)
def downloadSingleFile(self, filePath, syncPath=None):
"""download single file from Baidu Cloud"""
super(BaiduCloudAPI, self).downloadSingleFile(filePath, syncPath)
def deleteSingleFile(self, filePath, syncPath=None):
"""delete singel file or folder"""
super(BaiduCloudAPI, self).deleteSingleFile(filePath, syncPath)
param = {
'method': 'delete',
'access_token': self.cf.get("BaiduCloud", "access_token")
}
class BaiduCloudAPI(ClouldAPI):
"""Baidu Cloud API"""
def __init__(self, name):
super(BaiduCloudAPI, self).__init__(name)
self.cf = None
def installConf(self, conf):
"""install configuration parser"""
self.cf = conf
def applyAccess(self):
"""apply baidu access to netdisk"""
super(BaiduCloudAPI, self).applyAccess()
api_url = self.cf.get("BaiduCloud", "openapi_url") + "/device/code"
param = {
"client_id": self.cf.get("BaiduCloud", "api_key"),
"response_type": "device_code",
"scope": "basic netdisk"
}
logging.debug('[BaiduPlugin]: applyAccess: URL=> %s, API key=> %s',
api_url, self.cf.get("BaiduCloud", "api_key"))
ret_fp = cloud_get(api_url, param)
data = json.load(ret_fp)
webbrowser.open(data["verification_url"])
print "Please enter your user code %s into open web page" % data[
"user_code"]
print "Please note it will expires in %d s" % data["expires_in"]
ret_fp.close()
self.cf.set("BaiduCloud", "dev_code", data["device_code"])
def getToken(self):
"""Get access Token from Baidu API"""
super(BaiduCloudAPI, self).getToken()
api_url = self.cf.get("BaiduCloud", "openapi_url") + "/token"
logging.debug('[BaiduPlugin]: getToken: URL=>% s', api_url)
param = {
"grant_type": "device_token",
"code": self.cf.get("BaiduCloud", "dev_code"),
"client_id": self.cf.get("BaiduCloud", "api_key"),
"client_secret": self.cf.get("BaiduCloud", "secret_key")
}
ret_fp = cloud_get(api_url, param)
data = json.load(ret_fp)
logging.debug(
'[BaiduPlugin]: getToken: access_token=>%s, refresh_token=>%s',
data["access_token"], data["refresh_token"])
ret_fp.close()
self.cf.set("BaiduCloud", "access_token", data["access_token"])
self.cf.set("BaiduCloud", "refresh_token", data["refresh_token"])
self.cf.set("BaiduCloud", "session_key", data["session_key"])
self.cf.set("BaiduCloud", "session_secret", data["session_secret"])
self.cf.set("BaiduCloud", "scope", data["scope"])
def refreshToken(self):
"""Refresh access token"""
super(BaiduCloudAPI, self).refreshToken()
api_url = self.cf.get("BaiduCloud", "openapi_url") + "/token"
logging.debug('[BaiduPlugin]: refreshToken: URL=>%s', api_url)
param = {
"grant_type": "refresh_token",
"refresh_token": self.cf.get("BaiduCloud", "refresh_token"),
"client_id": self.cf.get("BaiduCloud", "api_key"),
"client_secret": self.cf.get("BaiduCloud", "secret_key")
}
try:
ret_fp = cloud_post(api_url, param)
data = json.load(ret_fp)
logging.debug(
'[BaiduPlugin]: getToken: access_token=>%s, refresh_token=>%s',
data["access_token"], data["refresh_token"])
self.cf.set("BaiduCloud", "access_token", data["access_token"])
self.cf.set("BaiduCloud", "refresh_token", data["refresh_token"])
self.cf.set("BaiduCloud", "session_key", data["session_key"])
self.cf.set("BaiduCloud", "session_secret", data["session_secret"])
self.cf.set("BaiduCloud", "scope", data["scope"])
ret_fp.close()
except ValueError, e:
logging.error('[%s]: refreshToken JSON load error %s', e)
except urllib2.HTTPError, e:
emsg = e.read()
logging.error('[%s]: HTTP error=>%d, result=>%s', self.getName(),
e.code, emsg)
edata = json.loads(emsg)
error = {
'http': e.code,
'code': edata['error'],
'description': edata['error_description']
}
self.errorHandler(error)
param = {'method': 'upload', 'ondup': onDup, 'access_token': self.cf.get("BaiduCloud", "access_token")}
param['path'] = self.cf.get("BaiduCloud", "app_path") + "/" + syncPath
res = E_API_ERR
try:
fp = open(filePath)
ret_fp = cloud_multi_post(self.cf.get("BaiduCloud", "upload_url"), param, {'file': fp})
res = json.load(ret_fp)
ret_fp.close()
fp.close()
except urllib2.HTTPError, e:
self.pcsErrorHandler(e)
res = E_API_ERR
except ValueError, e:
logging.error('[%s]: uploadSingleFile JSON load error %s', e)
res = E_VALUE_ERR
return self.parseResult(res)
def downloadSingleFile(self, filePath, syncPath=None):
"""download single file from Baidu Cloud"""
super(BaiduCloudAPI, self).downloadSingleFile(filePath, syncPath)
def deleteSingleFile(self, filePath, syncPath=None):
"""delete singel file or folder"""
super(BaiduCloudAPI, self).deleteSingleFile(filePath, syncPath)
param = {'method': 'delete', 'access_token': self.cf.get("BaiduCloud", "access_token")}
param['path'] = self.cf.get('BaiduCloud', 'app_path') + '/' + filePath
url = self.cf.get('BaiduCloud', 'pcs_url') + '/file'