def downloadToFileFromHTTPFormSubmit(self, url, method, paramDict, outputFile): self.httpLock.acquire() self.setupHandle() url = urllib.unquote(url) self.httpHeaders = StringIO.StringIO() oFile = None try: oFile = open(outputFile, 'wb') except: #unable to create a file for storing content self.httpLock.release() return '-2' if method == 'get': self.httpHandle.setopt(pycurl.URL, url + '?' + urllib.urlencode(paramDict)) else: self.httpHandle.setopt(pycurl.URL, url) self.httpHandle.setopt(pycurl.POST, 1) self.httpHandle.setopt(pycurl.POSTFIELDS, urllib.urlencode(paramDict)) self.httpHandle.setopt(pycurl.WRITEDATA, oFile) try: self.httpHandle.perform() except pycurl.error, e: #exception in peforming HTTP action logManager.log('Exception in retrieving url ' + url, logging.WARN) logManager.log(str(e[0]) + ' :: ' + str(e[1]), logging.DEBUG) oFile.close() self.setupHandle() self.httpLock.release() return '-1'
def registerMeeting(self, dimdimID, bookmarkAction): # FORMAT: {dimdimID: {bmAction: bookmarkAction, currentResource: resourceID, locked : False, resources : {RESOURCEDICT}}} ## bmAction is required for bookmarking functionality. stateMachine doesn't care what the action is. ## The caller should make sure it is valid and handle it appropriately. ## Default Values : currentResource : None, locked : False, resources : {'regular':{}, 'preloaded':{}} # NOTE: registerMeeting is for external usage only. importStateEngine shouldn't use registerMeeting # If there is a meeting that is already registered, registerMeeting ignores any further registrations if self.getMeeting(dimdimID): return meetingPath = os.path.join(os.path.join(osconfig.cobArchive(), dimdimID)) if os.path.isdir(meetingPath): try: shutil.rmtree(meetingPath) except: pass os.makedirs(meetingPath) self.stateLock.acquire() meeting = {dimdimID : {'bmAction' : bookmarkAction, 'currentResource' : None, 'locked' : False, 'resources' : {'regular':{}, 'preloaded':{}}}} self.stateMachine.update(meeting) self.stateLock.release() self.exportStateMachine(dimdimID) logManager.log('meeting ' + dimdimID + ' registered', logging.INFO) return
def registerResource(self, dimdimID, resourceID, resourceType): # FORMAT: resourceList is of the form {'regular':{RESOURCE1, RESOURCE2, ...}, 'preloaded':{RESOURCE3, RESOURCE4, ....}} # RESOURCE is of the form {resourceID : {lastPage: pageID, currentPage: pageID, pages:{PAGEDICT}}} ## Default values for RESOURCE: lastPage = -1, currentPage = -1 and {} as pages ## resourceType has to be 'regular' or 'preloaded'. # If a resource is already registered, we return immediately. # registerResource *must* not be called before registerMeeting. State Machine is not going to expect orphan resources # State Machine also doesn't expect any resourceType other than 'regular' or 'preloaded'. if self.getResourceByID(dimdimID, resourceID): return meeting = self.getMeeting(dimdimID) resourceList = meeting.get('resources') typeList = resourceList.get(resourceType) resource = { resourceID: { 'lastPage': -1, 'currentPage': -1, 'pages': {} } } self.stateLock.acquire() typeList.update(resource) meeting.update({'currentResource': resourceID}) self.stateLock.release() self.exportStateMachine(dimdimID) logManager.log( 'resource ' + resourceID + ' registered under meeting ' + dimdimID, logging.INFO) return
def registerResource(self, dimdimID, resourceID, resourceType): # FORMAT: resourceList is of the form {'regular':{RESOURCE1, RESOURCE2, ...}, 'preloaded':{RESOURCE3, RESOURCE4, ....}} # RESOURCE is of the form {resourceID : {lastPage: pageID, currentPage: pageID, pages:{PAGEDICT}}} ## Default values for RESOURCE: lastPage = -1, currentPage = -1 and {} as pages ## resourceType has to be 'regular' or 'preloaded'. # If a resource is already registered, we return immediately. # registerResource *must* not be called before registerMeeting. State Machine is not going to expect orphan resources # State Machine also doesn't expect any resourceType other than 'regular' or 'preloaded'. if self.getResourceByID(dimdimID, resourceID): return meeting = self.getMeeting(dimdimID) resourceList = meeting.get('resources') typeList = resourceList.get(resourceType) resource = {resourceID : {'lastPage' : -1, 'currentPage' : -1, 'pages' : {}}} self.stateLock.acquire() typeList.update(resource) meeting.update({'currentResource' : resourceID}) self.stateLock.release(); self.exportStateMachine(dimdimID) logManager.log('resource ' + resourceID + ' registered under meeting ' + dimdimID, logging.INFO) return
def handleMeetingClose(self, dimdimID): # when a meeting closes, all 'regular' resources # should be cleaned up both from the disk and the stateMachine logManager.log('meeting ' + dimdimID + ' is getting closed', logging.INFO) meeting = self.getMeeting(dimdimID) if not meeting: return resourceList = meeting.get('resources') preloadedList = resourceList.get('preloaded') if len(preloadedList) == 0: # get rid of the entire meeting record self.unregisterMeeting(dimdimID) try: # get rid of meta data os.remove( os.path.join( os.path.join(osconfig.cobArchive(), 'META-INF'), dimdimID)) except: pass try: # get rid of the files shutil.rmtree(os.path.join(osconfig.cobArchive(), dimdimID)) except: pass return self.stateLock.acquire() resourceList.update({'regular': {}}) self.stateLock.release() try: # delete the 'regular' folder shutil.rmtree( os.path.join(os.path.join(osconfig.cobArchive(), dimdimID), 'regular')) except: pass self.exportStateMachine(dimdimID) return
def importStateMachine(self): metaDir = os.path.join(osconfig.cobArchive(), 'META-INF') if not os.path.isdir(metaDir): return dimdimIDList = os.listdir(metaDir) for dimdimID in dimdimIDList: try: logManager.log('Detected meeting ' + dimdimID, logging.INFO) metaFileLocation = os.path.join(metaDir, dimdimID) metaFileHandle = open(metaFileLocation, 'rb') meeting = self.jsonObj.decode(metaFileHandle.read()) metaFileHandle.close() self.stateMachine.update({dimdimID : meeting}) except: logManager.log('Exception in importing state information of meeting ' + dimdimID, logging.WARN) pass return
def registerPage(self, dimdimID, associatedURL, events = {}): # associatedURL should be a valid http url # events is expected to be a dict type and can have multiple event types. # this routine creates a new page and updates the currentPage for the currentResource in dimdimID resource = self.getCurrentResource(dimdimID) pageList = resource.get('pages') currentPage = resource.get('currentPage') + 1 page = {'url' : associatedURL, 'events' : events} self.stateLock.acquire() pageList.update({currentPage : page}) resource.update({'lastPage' : currentPage, 'currentPage' : currentPage}) self.stateLock.release() self.exportStateMachine(dimdimID) logManager.log('url ' + associatedURL + ' has been added to current resource of meeting ' + dimdimID, logging.INFO) return currentPage
def syncToResource(self, dimdimID, roomID, sessionID, resourceID, returnType): # navigate to the given resource first self.stateEngine.navigateToResource(dimdimID, resourceID) self.stateEngine.resetCurrentResource(dimdimID) self.stateEngine.trimResourceToCurrentPage(dimdimID) # prepare attendee and presenter cache with current state resource = self.stateEngine.getCurrentResource(dimdimID) resourceType = self.stateEngine.getCurrentResourceType(dimdimID) currentPageNum = resource.get('currentPage') # prepare cache presenterLocation = resourceType + '/' + resourceID + '/' + str(currentPageNum) + '/presenter/content.html' presenterCache = string.replace(self.presenterCacheBuffer, 'DIMDIMSL_COB_CONTENT_SRC', presenterLocation) attendeeLocation = resourceType + '/' + resourceID + '/' + str(currentPageNum) + '/attendee/content.html' attendeeCache = string.replace(self.attendeeCacheBuffer, 'DIMDIMSL_COB_CONTENT_SRC', attendeeLocation) pCacheLocation = os.path.join(os.path.join(osconfig.cobArchive(), dimdimID), 'p_cache.html') pCacheHandle = open(pCacheLocation, 'wb') pCacheHandle.write(presenterCache) pCacheHandle.close() aCacheLocation = os.path.join(os.path.join(osconfig.cobArchive(), dimdimID), 'a_cache.html') aCacheHandle = open(aCacheLocation, 'wb') aCacheHandle.write(attendeeCache) aCacheHandle.close() if returnType == 'method': return 'cobCallback(\''+str(dimdimID) + '\')' jsonSuccess = jsonObject() jsonSuccess.clearResponse() jsonSuccess.add('result', 'true') jsonSuccess.add('method', 'syncToURLResource') jsonSuccess.add('error', '7200') jsonSuccess.add('location', '/content/' + str(dimdimID)) logManager.log('Meeting ' + dimdimID + ' synced to resource ' + resourceID, logging.INFO) return jsonSuccess.jsonResponse()
def importStateMachine(self): metaDir = os.path.join(osconfig.cobArchive(), 'META-INF') if not os.path.isdir(metaDir): return dimdimIDList = os.listdir(metaDir) for dimdimID in dimdimIDList: try: logManager.log('Detected meeting ' + dimdimID, logging.INFO) metaFileLocation = os.path.join(metaDir, dimdimID) metaFileHandle = open(metaFileLocation, 'rb') meeting = self.jsonObj.decode(metaFileHandle.read()) metaFileHandle.close() self.stateMachine.update({dimdimID: meeting}) except: logManager.log( 'Exception in importing state information of meeting ' + dimdimID, logging.WARN) pass return
def handleMeetingClose(self, dimdimID): # when a meeting closes, all 'regular' resources # should be cleaned up both from the disk and the stateMachine logManager.log('meeting ' + dimdimID + ' is getting closed', logging.INFO) meeting = self.getMeeting(dimdimID) if not meeting: return resourceList = meeting.get('resources') preloadedList = resourceList.get('preloaded') if len(preloadedList) == 0: # get rid of the entire meeting record self.unregisterMeeting(dimdimID) try: # get rid of meta data os.remove(os.path.join(os.path.join(osconfig.cobArchive(), 'META-INF'), dimdimID)) except: pass try: # get rid of the files shutil.rmtree(os.path.join(osconfig.cobArchive(), dimdimID)) except: pass return self.stateLock.acquire() resourceList.update({'regular':{}}) self.stateLock.release() try: # delete the 'regular' folder shutil.rmtree(os.path.join(os.path.join(osconfig.cobArchive(), dimdimID), 'regular')) except: pass self.exportStateMachine(dimdimID) return
def registerPage(self, dimdimID, associatedURL, events={}): # associatedURL should be a valid http url # events is expected to be a dict type and can have multiple event types. # this routine creates a new page and updates the currentPage for the currentResource in dimdimID resource = self.getCurrentResource(dimdimID) pageList = resource.get('pages') currentPage = resource.get('currentPage') + 1 page = {'url': associatedURL, 'events': events} self.stateLock.acquire() pageList.update({currentPage: page}) resource.update({'lastPage': currentPage, 'currentPage': currentPage}) self.stateLock.release() self.exportStateMachine(dimdimID) logManager.log( 'url ' + associatedURL + ' has been added to current resource of meeting ' + dimdimID, logging.INFO) return currentPage
def registerMeeting(self, dimdimID, bookmarkAction): # FORMAT: {dimdimID: {bmAction: bookmarkAction, currentResource: resourceID, locked : False, resources : {RESOURCEDICT}}} ## bmAction is required for bookmarking functionality. stateMachine doesn't care what the action is. ## The caller should make sure it is valid and handle it appropriately. ## Default Values : currentResource : None, locked : False, resources : {'regular':{}, 'preloaded':{}} # NOTE: registerMeeting is for external usage only. importStateEngine shouldn't use registerMeeting # If there is a meeting that is already registered, registerMeeting ignores any further registrations if self.getMeeting(dimdimID): return meetingPath = os.path.join( os.path.join(osconfig.cobArchive(), dimdimID)) if os.path.isdir(meetingPath): try: shutil.rmtree(meetingPath) except: pass os.makedirs(meetingPath) self.stateLock.acquire() meeting = { dimdimID: { 'bmAction': bookmarkAction, 'currentResource': None, 'locked': False, 'resources': { 'regular': {}, 'preloaded': {} } } } self.stateMachine.update(meeting) self.stateLock.release() self.exportStateMachine(dimdimID) logManager.log('meeting ' + dimdimID + ' registered', logging.INFO) return
def downloadToFileFromHTTPURL(self, url, outputFile): self.httpLock.acquire() self.setupHandle() url = urllib.unquote(url) # first check if this is a valid URL socketDataList = None try: urlData = urlparse.urlparse(url) except: logManager.log('failed to lookup hostname of ' + url, logging.WARN) self.httpLock.release() return '-1' self.httpHeaders = StringIO.StringIO() oFile = None try: oFile = open(outputFile, 'wb') except: #unable to create a file for storing content self.httpLock.release() return '-2' self.httpHandle.setopt(pycurl.URL, url) self.httpHandle.setopt(pycurl.WRITEDATA, oFile) try: self.httpHandle.perform() except pycurl.error, e: #exception in peforming HTTP action logManager.log('Exception in retrieving url ' + url, logging.WARN) logManager.log(str(e[0]) + ' :: ' + str(e[1]), logging.DEBUG) oFile.close() self.setupHandle() self.httpLock.release() return '-1'
# simply iterate till the end and use the # final location and the original URL flURL = item.group() flURL = string.replace(flURL, ' ', '') flURL = string.lstrip(flURL, 'Location:') self.httpLock.release() if string.find(flURL, 'http://') != 0 and string.find( flURL, 'https://') != 0: # this is a relative link. # create absolute using the current url # and call the function again. baseURL = string.rstrip(urlparse.urljoin(url, 'dimdimsl'), 'dimdimsl') flURL = urlparse.urljoin(baseURL, flURL) oFile.close() os.remove(outputFile) flURL = self.downloadToFileFromHTTPURL(flURL, outputFile) return flURL logManager.log('unable to fetch url ' + url, logging.WARN) logManager.log(headerData, logging.DEBUG) try: os.remove(outputFile) except: pass self.httpLock.release() return '0'
class CurlWrapper(object): httpHandle = None httpHeaders = None httpLock = None def __init__(self): self.httpHeaders = None self.httpLock = threading.BoundedSemaphore(1) #self.setupHandle() def __del__(self): self.httpHandle.close() del self.httpHandle def setupHandle(self): try: if self.httpHandle: self.httpHandle.close() except: pass try: del self.httpHandle except: pass self.httpHandle = None self.httpHandle = pycurl.Curl() self.httpHandle.setopt(pycurl.HTTPHEADER, curlHeaders()) self.httpHandle.setopt(pycurl.USERAGENT, curlUserAgent()) self.httpHandle.setopt(pycurl.FOLLOWLOCATION, 1) self.httpHandle.setopt(pycurl.CONNECTTIMEOUT, curlConnectTimeout()) self.httpHandle.setopt(pycurl.TIMEOUT, curlTimeout()) #self.httpHandle.setopt(pycurl.COOKIEJAR, 'cookiejar.txt') #self.httpHandle.setopt(pycurl.COOKIEFILE, 'cookiejar.txt') #self.httpHandle.setopt(pycurl.COOKIELIST, 'ALL') self.httpHandle.setopt(pycurl.HEADERFUNCTION, self.headersCallback) def headersCallback(self, buf): # Callback function invoked when header data is ready if self.httpHeaders: self.httpHeaders.write(buf) return def downloadToFileFromHTTPFormSubmit(self, url, method, paramDict, outputFile): self.httpLock.acquire() self.setupHandle() url = urllib.unquote(url) self.httpHeaders = StringIO.StringIO() oFile = None try: oFile = open(outputFile, 'wb') except: #unable to create a file for storing content self.httpLock.release() return '-2' if method == 'get': self.httpHandle.setopt(pycurl.URL, url + '?' + urllib.urlencode(paramDict)) else: self.httpHandle.setopt(pycurl.URL, url) self.httpHandle.setopt(pycurl.POST, 1) self.httpHandle.setopt(pycurl.POSTFIELDS, urllib.urlencode(paramDict)) self.httpHandle.setopt(pycurl.WRITEDATA, oFile) try: self.httpHandle.perform() except pycurl.error, e: #exception in peforming HTTP action logManager.log('Exception in retrieving url ' + url, logging.WARN) logManager.log(str(e[0]) + ' :: ' + str(e[1]), logging.DEBUG) oFile.close() self.setupHandle() self.httpLock.release() return '-1' oFile.flush() oFile.close() self.httpHeaders.flush() headerData = self.httpHeaders.getvalue() self.httpHeaders.close() del self.httpHeaders self.httpHeaders = None if string.find(headerData, '200 OK') > 0 and string.find( headerData, 'text/html') > 0: matchObj = re.finditer(('Location\s{0,}:\s{0,}\S+'), headerData) flURL = url for item in matchObj: # simply iterate till the end and use the # final location and the original URL flURL = item.group() flURL = string.replace(flURL, ' ', '') flURL = string.lstrip(flURL, 'Location:') self.httpLock.release() if string.find(flURL, 'http://') != 0 and string.find( flURL, 'https://') != 0: # this is a relative link. # create absolute using the current url # and call the function again. baseURL = string.rstrip(urlparse.urljoin(url, 'dimdimsl'), 'dimdimsl') flURL = urlparse.urljoin(baseURL, flURL) flURL = self.downloadToFileFromHTTPURL(flURL, outputFile) return flURL logManager.log('unable to fetch url ' + url, logging.WARN) logManager.log(headerData, logging.DEBUG) try: os.remove(outputFile) except: pass self.setupHandle() self.httpLock.release() return '0'
flURL = url for item in matchObj: # simply iterate till the end and use the # final location and the original URL flURL = item.group() flURL = string.replace(flURL, ' ', '') flURL = string.lstrip(flURL, 'Location:') self.httpLock.release() if string.find(flURL, 'http://') != 0 and string.find(flURL, 'https://') != 0: # this is a relative link. # create absolute using the current url # and call the function again. baseURL = string.rstrip(urlparse.urljoin(url, 'dimdimsl'), 'dimdimsl') flURL = urlparse.urljoin(baseURL, flURL) oFile.close() os.remove(outputFile) flURL = self.downloadToFileFromHTTPURL(flURL, outputFile) return flURL logManager.log('unable to fetch url ' + url, logging.WARN) logManager.log(headerData, logging.DEBUG) try: os.remove(outputFile) except: pass self.httpLock.release() return '0'
def createURLResource(self, dimdimID, roomID, sessionID, encodedURL, resourceID = '_default', resourceType = 'regular'): # createURLResource is only called for a fresh resource jsonError = jsonObject() jsonError.clearResponse() jsonError.add('result', 'false') jsonError.add('method', 'cacheURL') jsonError.add('error', '7500') meeting = self.stateEngine.getMeeting(dimdimID) if not meeting: # register the meeting first # ideally, this call should never come.. # meeting should be registered in listURLResources logManager.log('Detected that meeting ' + dimdimID + ' was never registered. This could mean that listURLResources was not called', logging.WARN) self.stateEngine.registerMeeting(dimdimID, '') self.stateEngine.registerMeeting(dimdimID, 'http://webmeeting.dimdim.com:80/dimdim/BookmarkCobResource.action') #return jsonError.jsonResponse() if resourceType != 'regular': resourceType = 'preloaded' if resourceID == '_default': resourceType = 'regular' # can't have preloaded resources with _default originalURL = urllib.unquote(encodedURL) baseURL = self.baseURL(originalURL) pageNum = '0' # duh! this is a new resource resourcePath = os.path.join(os.path.join(os.path.join(osconfig.cobArchive(), dimdimID), resourceType), resourceID) presenterPagePath = os.path.join(os.path.join(resourcePath, pageNum), 'presenter') attendeePagePath = os.path.join(os.path.join(resourcePath, pageNum), 'attendee') if not os.path.isdir(presenterPagePath): os.makedirs(presenterPagePath) if not os.path.isdir(attendeePagePath): os.makedirs(attendeePagePath) tempStore = os.path.join(resourcePath, 'temp.html') try: os.remove(tempStore) except: pass retval = self.curlHandle.downloadToFileFromHTTPURL(encodedURL, tempStore) if len(retval) < 3: try: shutil.rmtree(presenterPagePath) shutil.rmtree(attendeePagePath) os.remove(tempStore) except: pass return jsonError.jsonResponse() if originalURL != retval: # must be a 302. this is our new URL originalURL = retval baseURL = self.baseURL(originalURL) encodedURL = urllib.quote(originalURL) htmlHandle = open(tempStore, 'r') htmlContent = htmlHandle.read() htmlHandle.close() # inject baseURL htmlContent = contentMgr.sanitizeHTML(htmlContent, baseURL) self.prepareHtml(originalURL, htmlContent, presenterPagePath, attendeePagePath) # prepare cache presenterLocation = resourceType + '/' + resourceID + '/' + pageNum + '/presenter/content.html' presenterCache = string.replace(self.presenterCacheBuffer, 'DIMDIMSL_COB_CONTENT_SRC', presenterLocation) attendeeLocation = resourceType + '/' + resourceID + '/' + pageNum + '/attendee/content.html' attendeeCache = string.replace(self.attendeeCacheBuffer, 'DIMDIMSL_COB_CONTENT_SRC', attendeeLocation) pCacheLocation = os.path.join(os.path.join(osconfig.cobArchive(), dimdimID), 'p_cache.html') pCacheHandle = open(pCacheLocation, 'wb') pCacheHandle.write(presenterCache) pCacheHandle.close() aCacheLocation = os.path.join(os.path.join(osconfig.cobArchive(), dimdimID), 'a_cache.html') aCacheHandle = open(aCacheLocation, 'wb') aCacheHandle.write(attendeeCache) aCacheHandle.close() # clean up try: os.remove(tempStore) except: pass # register everything self.stateEngine.registerResource(dimdimID, resourceID, resourceType) self.stateEngine.registerPage(dimdimID, originalURL) jsonSuccess = jsonObject() jsonSuccess.clearResponse() jsonSuccess.add('result', 'true') jsonSuccess.add('method', 'cacheURL') jsonSuccess.add('error', '7200') jsonSuccess.add('location', '/content/' + str(dimdimID)) return jsonSuccess.jsonResponse()