def pcsErrorHandler(self, e): """pcs error handler""" emsg = e.read() logging.error('[%s]: HTTP error=>%d, result=>%s', self.getName(), e.code, emsg) edata = json.loads(emsg) error = {'http': e.code, 'code': edata['error_code']} self.errorHandler(error)
def send(self, msg): """send message to related msg Queue""" q = self.findQ(msg.header.rUid) if q: q.put(msg) else: logging.error('[UMsgBus]: send: find rUid %d queue failure', msg.header.rUid)
def refreshToken(self): """Refresh access token""" super(BaiduCloudAPI, self).refreshToken() api_url = self.cf.get("BaiduCloud", "openapi_url") + "/token" logging.debug('[BaiduPlugin]: refreshToken: URL=>%s', api_url) param = { "grant_type": "refresh_token", "refresh_token": self.cf.get("BaiduCloud", "refresh_token"), "client_id": self.cf.get("BaiduCloud", "api_key"), "client_secret": self.cf.get("BaiduCloud", "secret_key") } try: ret_fp = cloud_post(api_url, param) data = json.load(ret_fp) logging.debug( '[BaiduPlugin]: getToken: access_token=>%s, refresh_token=>%s', data["access_token"], data["refresh_token"]) self.cf.set("BaiduCloud", "access_token", data["access_token"]) self.cf.set("BaiduCloud", "refresh_token", data["refresh_token"]) self.cf.set("BaiduCloud", "session_key", data["session_key"]) self.cf.set("BaiduCloud", "session_secret", data["session_secret"]) self.cf.set("BaiduCloud", "scope", data["scope"]) ret_fp.close() except ValueError, e: logging.error('[%s]: refreshToken JSON load error %s', e)
def loadAllPlugins(self): """load all plugins from destant path""" p_paths = ConfManager.getManager().getPluginPaths() pluginList = ConfManager.getManager().getValue('common', 'plugins') for p in p_paths: logging.debug('[%s]: loadAllPlugins from %s', 'PluginManager', p) #TODO: __import__ the module into our script try: dirs = os.listdir(p) for d in dirs: tmp = '%s%s%s' % (p, os.sep, d) if os.path.isdir(tmp): module_name = 'UniFileSync.plugins.%s' % d module_path = '%s%s%sPlugin.py' % (tmp, os.sep, d) imp.load_source('', module_path) pl = {'name': d, 'path': module_path} if pl not in pluginList: pluginList.append(pl) except OSError as exc: logging.error('loadAllPlugins listdir error %d', OSError.errno) ConfManager.getManager().setValue('common', 'plugins', pluginList) logging.debug('[%s]: loadAllPlugins and save %s into configuration', 'PluginManager', pluginList) ConfManager.getManager().save()
def run(self): """UServer entry""" super(UServer, self).run() #TODO: below should be set by ConfManager if self.isEnableSocket: self.sock.bind(('localhost', 8089)) self.sock.listen(5) while self.isRunning: try: conn, addr = self.sock.accept() try: conn.settimeout(5) buf = conn.recv(1024) #TODO: should be also in ConfManager req = json.loads(buf) logging.debug('[UniFileSync]: action %s, param %s', req['action'], req['param']) #TODO: make it common for function usage res, data = self.getHandler(req['action'])(req['param']) ret = {'action': req['action'], 'param': {'data': data}, 'res': res, 'type': 'ack'} conn.send(json.dumps(ret)) except socket.timeout: logging.info('[UniFileSync]: socket time out from %s', addr) except KeyError, e: logging.error('[%s]: Key Error with param %s', self.getName(), req['param']) finally: conn.close() except KeyboardInterrupt: print 'Press Ctrl+C' self.stop()
def getSyncFolder(self, path): """get sync folder from path""" dirpath = path try: index = path.rindex(os.sep) + 1 dirpath = path[index:] except ValueError, KeyError: logging.error('[%s]: getSyncFolder failure with no so seperator', self.getName())
def regUniID(self, msgUniID): """register new msg uni ID""" if msgUniID not in self.__uni_ID_list: MsgBus.__lock.acquire() self.__uni_ID_list.append(msgUniID) MsgBus.__lock.release() return E_OK logging.error('regUniID failure due to Unique Message ID %d already registered', msgUniID) return E_INVILD_PARAM
def regQ(self, msgUniID, msgQueue): """register msg Queue""" if msgUniID not in self.__uni_ID_list: logging.error('regQ failure due to no Unique Message ID registered') return E_INVILD_PARAM UMsgBus.__lock.acquire() self.__queue_table[msgUniID] = msgQueue UMsgBus.__lock.release() return E_OK
def broadcast(self, msg): """broadcast message to all listeners""" sUid = msg.header.sUid for l in self.getListeners(sUid): q = self.findQ(l) if q: q.put(msg) else: logging.error('[UMsgBus]: broadcast: find sUid %d listener %d queue failure', msg.header.sUid, l)
def regQ(self, msgUniID, msgQueue): """register msg Queue""" if msgUniID not in self.__uni_ID_list: logging.error( 'regQ failure due to no Unique Message ID registered') return E_INVILD_PARAM UMsgBus.__lock.acquire() self.__queue_table[msgUniID] = msgQueue UMsgBus.__lock.release() return E_OK
def register_openers(): """register some openers into urlib2""" #Enable media post, proxy, cookie #logging.debug('%s', __handlers) res = E_OK try: urllib2.install_opener(urllib2.build_opener(*__handlers)) except Exception, e: logging.error('[register_openers]: exception occurs %s', e) res = E_OPEN_FAIL
def msgLoop(self): """message loop for processing message""" while self.__isRunning: try: msg = self.__msgQueue.get(True) if msg and msg.header.mtype: self.__operTable[msg.header.mtype](msg) except Queue.Empty, e: logging.error('[%s]: msgLoop timeout in %d with empty item', self.getName(), timeout) finally:
def broadcast(self, msg): """broadcast message to all listeners""" sUid = msg.header.sUid for l in self.getListeners(sUid): q = self.findQ(l) if q: q.put(msg) else: logging.error( '[UMsgBus]: broadcast: find sUid %d listener %d queue failure', msg.header.sUid, l)
def regUniID(self, msgUniID): """register new msg uni ID""" if msgUniID not in self.__uni_ID_list: MsgBus.__lock.acquire() self.__uni_ID_list.append(msgUniID) MsgBus.__lock.release() return E_OK logging.error( 'regUniID failure due to Unique Message ID %d already registered', msgUniID) return E_INVILD_PARAM
def getMsg(self, timeout=0): """process message within timeout""" msg = None try: msg = self.__msgQueue.get(True, timeout) except Queue.Empty: logging.error('[%s]: getMsg timeout in %d with empty item', self.getName(), timeout) finally: pass return msg
def startHandler(self, param): """start handler for actors""" logging.debug('[%s]: startHandler with parm %s', self.getName(), param) if param['name']: name = param['name'].lower() else: name = '' res = E_OK watchpaths = ConfManager.getManager().getValue('common', 'folders') if name == 'all' or name == '': if self.cActor.isRunning or self.fsMonitor.isRunning: return E_ACTOR_ALIVE, None self.cActor.start() self.__startActors.append(self.cActor) self.fsMonitor.start() self.__startActors.append(self.fsMonitor) time.sleep(2) for w in watchpaths: self.watchHandler({'path': w, 'action': 'add'}, False) #logging.debug('[%s]: add watch path %s from configuration file', self.getName(), w) elif name == 'monitor': if self.fsMonitor.isRunning: return E_ACTOR_ALIVE, None self.fsMonitor.start() self.__startActors.append(self.fsMonitor) time.sleep(2) for w in watchpaths: self.watchHandler({'path': w, 'action': 'add'}) logging.debug( '[%s]: add watch path %s from configuration file', self.getName(), w) elif name == 'cloud': if self.cActor.isRunning: return E_ACTOR_ALIVE, None self.cActor.start() self.__startActors.append(self.cActor) else: logging.error('[%s]: startHandler with name %s error', self.getName(), param['name']) res = E_INVILD_PARAM return res, None
def getCloudInfo(self): url = self.cf.get("BaiduCloud", "pcs_url") + "/quota" param = {"method": "info", "access_token": self.cf.get("BaiduCloud", "access_token")} res = E_API_ERR data = "getCloudInfo failure" try: f = cloud_get(url, param) data = json.load(f) res = E_OK f.close() except ValueError, e: logging.error('[%s]: getCloudInfo JSON load error %s', e) res = E_VALUE_ERR
def stopHandler(self, param): """stop handler for actors""" logging.debug('[%s]: stopHandler with parm %s', self.getName(), param) if param['name']: name = param['name'].lower() else: name = '' res = E_ACTOR_DEAD if name == 'all' or name == '': #The msg is broadcase msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, None, True) self.notifyListeners(msg) for a in self.__startActors: rmsg = self.getMsg(2) if rmsg: res = rmsg.body['result'] logging.info('[UniFileSync]: stop server...') self.stop() elif name == 'monitor': if not self.fsMonitor.isRunning: res = E_ACTOR_DEAD msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, MSG_UNIQUE_ID_T_FS_MONITOR, True) self.msgBus.send(msg) rmsg = self.getMsg(2) if rmsg: res = rmsg.body['result'] elif name == 'cloud': if not self.cActor.isRunning: res = E_ACTOR_DEAD msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, MSG_UNIQUE_ID_T_CLOUD_ACTOR, True) self.msgBus.send(msg) rmsg = self.getMsg(2) if rmsg: res = rmsg.body['result'] else: logging.error('[%s]: stopHandler with name %s error', self.getName(), param['name']) res = E_INVILD_PARAM return res, None
def startHandler(self, param): """start handler for actors""" logging.debug('[%s]: startHandler with parm %s', self.getName(), param); if param['name']: name = param['name'].lower() else: name = '' res = E_OK watchpaths = ConfManager.getManager().getValue('common', 'folders') if name == 'all' or name == '': if self.cActor.isRunning or self.fsMonitor.isRunning: return E_ACTOR_ALIVE, None self.cActor.start() self.__startActors.append(self.cActor) self.fsMonitor.start() self.__startActors.append(self.fsMonitor) time.sleep(2) for w in watchpaths: self.watchHandler({'path': w, 'action': 'add'}, False) #logging.debug('[%s]: add watch path %s from configuration file', self.getName(), w) elif name == 'monitor': if self.fsMonitor.isRunning: return E_ACTOR_ALIVE, None self.fsMonitor.start() self.__startActors.append(self.fsMonitor) time.sleep(2) for w in watchpaths: self.watchHandler({'path': w, 'action': 'add'}) logging.debug('[%s]: add watch path %s from configuration file', self.getName(), w) elif name == 'cloud': if self.cActor.isRunning: return E_ACTOR_ALIVE, None self.cActor.start() self.__startActors.append(self.cActor) else: logging.error('[%s]: startHandler with name %s error', self.getName(), param['name']) res = E_INVILD_PARAM return res, None
def getCloudInfo(self): url = self.cf.get("BaiduCloud", "pcs_url") + "/quota" param = { "method": "info", "access_token": self.cf.get("BaiduCloud", "access_token") } res = E_API_ERR data = "getCloudInfo failure" try: f = cloud_get(url, param) data = json.load(f) res = E_OK f.close() except ValueError, e: logging.error('[%s]: getCloudInfo JSON load error %s', e) res = E_VALUE_ERR
def stopHandler(self, param): """stop handler for actors""" logging.debug('[%s]: stopHandler with parm %s', self.getName(), param); if param['name']: name = param['name'].lower() else: name = '' res = E_ACTOR_DEAD if name == 'all' or name == '': #The msg is broadcase msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, None, True) self.notifyListeners(msg) for a in self.__startActors: rmsg = self.getMsg(2) if rmsg: res = rmsg.body['result'] logging.info('[UniFileSync]: stop server...') self.stop() elif name == 'monitor': if not self.fsMonitor.isRunning: res = E_ACTOR_DEAD msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, MSG_UNIQUE_ID_T_FS_MONITOR, True) self.msgBus.send(msg) rmsg = self.getMsg(2) if rmsg: res = rmsg.body['result'] elif name == 'cloud': if not self.cActor.isRunning: res = E_ACTOR_DEAD msg = self.initMsg(MSG_TYPE_T_OPER, MSG_ID_T_OPER_STOP, MSG_UNIQUE_ID_T_CLOUD_ACTOR, True) self.msgBus.send(msg) rmsg = self.getMsg(2) if rmsg: res = rmsg.body['result'] else: logging.error('[%s]: stopHandler with name %s error', self.getName(), param['name']) res = E_INVILD_PARAM return res, None
def run(self): """UServer entry""" super(UServer, self).run() #TODO: below should be set by ConfManager if self.isEnableSocket: self.sock.bind(('localhost', 8089)) self.sock.listen(5) while self.isRunning: try: conn, addr = self.sock.accept() try: conn.settimeout(5) buf = conn.recv( 1024) #TODO: should be also in ConfManager req = json.loads(buf) logging.debug('[UniFileSync]: action %s, param %s', req['action'], req['param']) #TODO: make it common for function usage res, data = self.getHandler(req['action'])( req['param']) ret = { 'action': req['action'], 'param': { 'data': data }, 'res': res, 'type': 'ack' } conn.send(json.dumps(ret)) except socket.timeout: logging.info('[UniFileSync]: socket time out from %s', addr) except KeyError, e: logging.error('[%s]: Key Error with param %s', self.getName(), req['param']) finally: conn.close() except KeyboardInterrupt: print 'Press Ctrl+C' self.stop()
def msgLoop(self): """message loop for server""" res = E_OK data = None while self.isRunning: try: msg = self.msgQueue.get(True) if msg and msg.header.mtype: #This is msg sent by FSMonitor and get result from CloudActor if 'result' in msg.body: for c in self.__callbackList: c(msg.body) continue res, data = self.getHandler(msg.header.mtype)(msg.body) if msg.header.ack: for c in self.__callbackList: c({'result': res, 'data': data}) except Queue.Empty, e: logging.error('[%s]: msgLoop timeout with empty item', self.getName()) finally:
def refreshToken(self): """Refresh access token""" super(BaiduCloudAPI, self).refreshToken() api_url = self.cf.get("BaiduCloud", "openapi_url") + "/token" logging.debug('[BaiduPlugin]: refreshToken: URL=>%s', api_url) param = {"grant_type": "refresh_token", "refresh_token": self.cf.get("BaiduCloud", "refresh_token"), "client_id": self.cf.get("BaiduCloud", "api_key"), "client_secret": self.cf.get("BaiduCloud", "secret_key")} try: ret_fp = cloud_post(api_url, param) data = json.load(ret_fp) logging.debug('[BaiduPlugin]: getToken: access_token=>%s, refresh_token=>%s', data["access_token"], data["refresh_token"]) self.cf.set("BaiduCloud", "access_token", data["access_token"]) self.cf.set("BaiduCloud", "refresh_token", data["refresh_token"]) self.cf.set("BaiduCloud", "session_key", data["session_key"]) self.cf.set("BaiduCloud", "session_secret", data["session_secret"]) self.cf.set("BaiduCloud", "scope", data["scope"]) ret_fp.close() except ValueError, e: logging.error('[%s]: refreshToken JSON load error %s', e)
def getAPI(self): """get plugin cloud API""" if not self.__API: logging.error(self.name + ' does not have any cloud API') return self.__API
param['path'] = self.cf.get("BaiduCloud", "app_path") + "/" + syncPath res = E_API_ERR try: fp = open(filePath) ret_fp = cloud_multi_post(self.cf.get("BaiduCloud", "upload_url"), param, {'file': fp}) res = json.load(ret_fp) ret_fp.close() fp.close() except urllib2.HTTPError, e: self.pcsErrorHandler(e) res = E_API_ERR except ValueError, e: logging.error('[%s]: uploadSingleFile JSON load error %s', e) res = E_VALUE_ERR return self.parseResult(res) def downloadSingleFile(self, filePath, syncPath=None): """download single file from Baidu Cloud""" super(BaiduCloudAPI, self).downloadSingleFile(filePath, syncPath) def deleteSingleFile(self, filePath, syncPath=None): """delete singel file or folder""" super(BaiduCloudAPI, self).deleteSingleFile(filePath, syncPath) param = { 'method': 'delete', 'access_token': self.cf.get("BaiduCloud", "access_token") }
class BaiduCloudAPI(ClouldAPI): """Baidu Cloud API""" def __init__(self, name): super(BaiduCloudAPI, self).__init__(name) self.cf = None def installConf(self, conf): """install configuration parser""" self.cf = conf def applyAccess(self): """apply baidu access to netdisk""" super(BaiduCloudAPI, self).applyAccess() api_url = self.cf.get("BaiduCloud", "openapi_url") + "/device/code" param = { "client_id": self.cf.get("BaiduCloud", "api_key"), "response_type": "device_code", "scope": "basic netdisk" } logging.debug('[BaiduPlugin]: applyAccess: URL=> %s, API key=> %s', api_url, self.cf.get("BaiduCloud", "api_key")) ret_fp = cloud_get(api_url, param) data = json.load(ret_fp) webbrowser.open(data["verification_url"]) print "Please enter your user code %s into open web page" % data[ "user_code"] print "Please note it will expires in %d s" % data["expires_in"] ret_fp.close() self.cf.set("BaiduCloud", "dev_code", data["device_code"]) def getToken(self): """Get access Token from Baidu API""" super(BaiduCloudAPI, self).getToken() api_url = self.cf.get("BaiduCloud", "openapi_url") + "/token" logging.debug('[BaiduPlugin]: getToken: URL=>% s', api_url) param = { "grant_type": "device_token", "code": self.cf.get("BaiduCloud", "dev_code"), "client_id": self.cf.get("BaiduCloud", "api_key"), "client_secret": self.cf.get("BaiduCloud", "secret_key") } ret_fp = cloud_get(api_url, param) data = json.load(ret_fp) logging.debug( '[BaiduPlugin]: getToken: access_token=>%s, refresh_token=>%s', data["access_token"], data["refresh_token"]) ret_fp.close() self.cf.set("BaiduCloud", "access_token", data["access_token"]) self.cf.set("BaiduCloud", "refresh_token", data["refresh_token"]) self.cf.set("BaiduCloud", "session_key", data["session_key"]) self.cf.set("BaiduCloud", "session_secret", data["session_secret"]) self.cf.set("BaiduCloud", "scope", data["scope"]) def refreshToken(self): """Refresh access token""" super(BaiduCloudAPI, self).refreshToken() api_url = self.cf.get("BaiduCloud", "openapi_url") + "/token" logging.debug('[BaiduPlugin]: refreshToken: URL=>%s', api_url) param = { "grant_type": "refresh_token", "refresh_token": self.cf.get("BaiduCloud", "refresh_token"), "client_id": self.cf.get("BaiduCloud", "api_key"), "client_secret": self.cf.get("BaiduCloud", "secret_key") } try: ret_fp = cloud_post(api_url, param) data = json.load(ret_fp) logging.debug( '[BaiduPlugin]: getToken: access_token=>%s, refresh_token=>%s', data["access_token"], data["refresh_token"]) self.cf.set("BaiduCloud", "access_token", data["access_token"]) self.cf.set("BaiduCloud", "refresh_token", data["refresh_token"]) self.cf.set("BaiduCloud", "session_key", data["session_key"]) self.cf.set("BaiduCloud", "session_secret", data["session_secret"]) self.cf.set("BaiduCloud", "scope", data["scope"]) ret_fp.close() except ValueError, e: logging.error('[%s]: refreshToken JSON load error %s', e) except urllib2.HTTPError, e: emsg = e.read() logging.error('[%s]: HTTP error=>%d, result=>%s', self.getName(), e.code, emsg) edata = json.loads(emsg) error = { 'http': e.code, 'code': edata['error'], 'description': edata['error_description'] } self.errorHandler(error)
param = {'method': 'upload', 'ondup': onDup, 'access_token': self.cf.get("BaiduCloud", "access_token")} param['path'] = self.cf.get("BaiduCloud", "app_path") + "/" + syncPath res = E_API_ERR try: fp = open(filePath) ret_fp = cloud_multi_post(self.cf.get("BaiduCloud", "upload_url"), param, {'file': fp}) res = json.load(ret_fp) ret_fp.close() fp.close() except urllib2.HTTPError, e: self.pcsErrorHandler(e) res = E_API_ERR except ValueError, e: logging.error('[%s]: uploadSingleFile JSON load error %s', e) res = E_VALUE_ERR return self.parseResult(res) def downloadSingleFile(self, filePath, syncPath=None): """download single file from Baidu Cloud""" super(BaiduCloudAPI, self).downloadSingleFile(filePath, syncPath) def deleteSingleFile(self, filePath, syncPath=None): """delete singel file or folder""" super(BaiduCloudAPI, self).deleteSingleFile(filePath, syncPath) param = {'method': 'delete', 'access_token': self.cf.get("BaiduCloud", "access_token")} param['path'] = self.cf.get('BaiduCloud', 'app_path') + '/' + filePath url = self.cf.get('BaiduCloud', 'pcs_url') + '/file'