def parseUBX(self, command, tid, params, payload): '''this function parses the UBX payload, and sets the personal message or current media''' pm = payload.split('<PSM>')[ 1 ].split('</PSM>')[ 0 ] pm = common.unescape(pm) self.contactManager.setContactPersonalMessage(tid, pm) if payload.find('<CurrentMedia>') != -1: media = payload.split('<CurrentMedia>')[1]\ .split('</CurrentMedia>')[0] mhead = media.find('\\0Music\\01\\0') if mhead != -1: # FIXME: OH GAWD PLEASE GET THIS OUTTA HERE media = '\xe2\x99\xab ' + media[mhead+12:] margs = media.split('\\0') media = margs[0] for args in range(1, len(margs)): media = media.replace('{%s}' % (args-1), margs[args]) media = common.unescape(media) self.contactManager.setContactPersonalMessage(tid, media) if self.lastCommand != 'ILN': #TODO: is this signal needed? self.emit('personal-message-changed', tid, pm) contact = self.contactManager.getContact(tid) self.emit('user-attr-changed', contact)
def onGetProfile(self, response): #print response.body if response.status[0] != 200 or \ response.body.find('<ExpressionProfile>') == -1: if self.triedCreatingProfile: # we already tried, but failed.. :( self.affinityCache = '' self.rid = '' else: #user doesn't have a roaming profile, try create one. soap.requests.create_profile(self.proxy, self.reRequestProfile) return try: nick = response.body.split('</DisplayName>')[0].split( '<DisplayName>')[1] nick = common.unescape(nick) except: nick = '' try: pm = response.body.split('</PersonalStatus>')[0].split( '<PersonalStatus>')[1] pm = common.unescape(pm) except: pm = '' try: dpurl = response.body.split('</StaticUserTilePublicURL>')[0].split( '<StaticUserTilePublicURL>')[1] photo = response.body.split('</Photo>')[0].split('<Photo>')[1] self.dpid = photo.split('</ResourceID>')[0].split( '<ResourceID>')[1] name = photo.split('</Name>')[0].split('Name>')[1] except: dpurl = '' self.dpid = '' if dpurl != '' and name != self.config.glob['Id']: gobject.idle_add(self.getPicture, dpurl) self.changeNick(nick, initial=True) self.changePersonalMessage(pm) try: self.affinityCache = response.body.split('</CacheKey>')[0].split( '<CacheKey>')[1] except: self.affinityCache = '' expression_profile = response.body.split( '</ExpressionProfile>')[0].split('<ExpressionProfile>')[1] self.rid = expression_profile.split('</ResourceID>')[0].split( '<ResourceID>')[1] self.profile_retrieved = True
def getNickFromDynamicItems(xml): try: nick = xml.split('<contactType>Me</contactType>')\ [1].split('</displayName>')[0].split('<displayName>')[1] return common.unescape(nick) except IndexError: return ''
def get_redirect(url, html): """Check for meta redirects and return redirect URL if found """ match = re.compile('<meta[^>]*?url=(.*?)["\']', re.IGNORECASE).search(html) if match: return urlparse.urljoin(url, common.unescape(match.groups()[0].strip()))
def normalize(link): """Normalize the link to avoid duplicates """ if '#' in link: # remove internal links to avoid duplicates link = link[:link.index('#')] link = common.unescape(link) # remove & from link return urlparse.urljoin(url, link) # support relative links
def run_command(tokens): """Execute a command line (treat internal and external appropriately""" # Cleanup environment for var in pseudo_vars: if var in os.environ.keys(): del os.environ[var] if tokens[0] == 'exit': internal_exit('Bye!') elif tokens[0].lower() == 'cd' and [t for t in tokens if t in sep_tokens ] == []: # This is a single CD command -- use our custom, more handy CD internal_cd([unescape(t) for t in tokens[1:]]) else: if set(sep_tokens).intersection(tokens) == set([]): # This is a simple (non-compound) command # Crude hack so that we return to the prompt when starting GUI # applications: if we think that the first token on the given command # line is an executable, check its PE header to decide whether it's # GUI application. If it is, spawn the process and then get on with # life. cmd = expand_env_vars(tokens[0].strip('"')) dir, name = os.path.split(cmd) ext = os.path.splitext(name)[1] if not ext or ext in exec_extensions: # Executable given app = cmd else: # Not an executable -- search for the associated application if os.path.isfile(cmd): app = associated_application(ext) else: # No application will be spawned if the file doesn't exist app = None if app: executable = full_executable_path(app) if executable and os.path.splitext( executable)[1].lower() == '.exe': # This is an exe file, try to figure out whether it's a GUI # or console application if is_gui_application(executable): import subprocess s = u' '.join([expand_tilde(t) for t in tokens]) subprocess.Popen(s.encode(sys.getfilesystemencoding()), shell=True) return # Regular (external) command start_time = time.time() run_in_cmd(tokens) console_window = win32console.GetConsoleWindow() if win32gui.GetForegroundWindow( ) != console_window and time.time() - start_time > 15: # If the window is inactive, flash after long tasks win32gui.FlashWindowEx(console_window, win32con.FLASHW_ALL, 3, 750)
def run_command(tokens): """Execute a command line (treat internal and external appropriately""" # Cleanup environment for var in pseudo_vars: if var in os.environ.keys(): del os.environ[var] if tokens[0] == 'exit': internal_exit('Bye!') elif is_pure_cd(tokens): # This is a single CD command -- use our custom, more handy CD internal_cd([unescape(t) for t in tokens[1:]]) else: if set(sep_tokens).intersection(tokens) == set([]): # This is a simple (non-compound) command # Crude hack so that we return to the prompt when starting GUI # applications: if we think that the first token on the given command # line is an executable, check its PE header to decide whether it's # GUI application. If it is, spawn the process and then get on with # life. cmd = expand_env_vars(tokens[0].strip('"')) dir, name = os.path.split(cmd) ext = os.path.splitext(name)[1] if not ext or ext in exec_extensions: # Executable given app = cmd else: # Not an executable -- search for the associated application if os.path.isfile(cmd): app = associated_application(ext) else: # No application will be spawned if the file doesn't exist app = None if app: executable = full_executable_path(app) if executable and os.path.splitext(executable)[1].lower() == '.exe': # This is an exe file, try to figure out whether it's a GUI # or console application if is_gui_application(executable): import subprocess s = u' '.join([expand_tilde(t) for t in tokens]) subprocess.Popen(s.encode(sys.getfilesystemencoding()), shell=True) return # Regular (external) command start_time = time.time() run_in_cmd(tokens) console_window = win32console.GetConsoleWindow() if win32gui.GetForegroundWindow() != console_window and time.time() - start_time > 15: # If the window is inactive, flash after long t1asks win32gui.FlashWindowEx(console_window, win32con.FLASHW_ALL, 3, 750)
def getNickFromDynamicItems(self, dynamicItems): # get the nick from the XML try: temp = dynamicItems.split('<contactType>Me</contactType>')\ [1].split('</displayName>')[0] nick = common.unescape(temp.split('<displayName>')[1]) except: nick = self.user if nick != self.nick: oldNick = self.nick self.nick = nick self.emit('self-nick-changed', oldNick, self.nick)
def gtrans_get(self, url, **kwargs): """Get page via Google Translation """ url = 'http://translate.google.com/translate?sl=nl&anno=2&u=%s' % urllib.quote(url) html = self.get(url, **kwargs) if html: m = re.compile(r'<frame src="([^"]+)" name=c>', re.DOTALL|re.IGNORECASE).search(html) if m: frame_src = urlparse.urljoin(url, common.unescape(m.groups()[0].strip())) # force to check redirect here if kwargs.has_key('num_redirects'): kwargs['num_redirects'] = 1 html = self.get(frame_src, **kwargs) if html: # remove google translations content return re.compile(r'<span class="google-src-text".+?</span>', re.DOTALL|re.IGNORECASE).sub('', html)
def gtrans_get(self, url, **kwargs): """Download webpage via Google Translation """ url = 'http://translate.google.com/translate?sl=nl&anno=2&u=%s' % urllib.quote(url) html = self.get(url, **kwargs) if html: m = re.compile(r'<iframe[^<>]*src="([^"]+)"[^<>]*name=c', re.DOTALL|re.IGNORECASE).search(html) if m: frame_src = urlparse.urljoin(url, common.unescape(m.groups()[0].strip())) # force to check redirect here html = self.get(frame_src, **kwargs) if html: # remove google translations content return re.compile(r'<span class="google-src-text".+?</span>', re.DOTALL|re.IGNORECASE).sub('', html) return self.settings.default
def get_excerpt(html, try_meta=False, max_chars=255): """Extract excerpt from this HTML by finding largest text block try_meta indicates whether to try extracting from meta description tag max_chars is the maximum number of characters for the excerpt """ # try extracting meta description tag excerpt = '' if try_meta: excerpt = xpath.get(html, '/html/head/meta[@name="description"]/@content') if not excerpt: # remove these tags and then find biggest text block bad_tags = 'hr', 'br', 'script', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6' content = common.remove_tags(xpath.get(html, '/html/body', remove=bad_tags)) if content: excerpt = max((len(p.strip()), p) for p in content.splitlines())[1] return common.unescape(excerpt.strip())[:max_chars]
def get_excerpt(html, try_meta=False, max_chars=10000): """Extract excerpt from this HTML by finding largest text block try_meta indicates whether to try extracting from meta description tag max_chars is the maximum number of characters for the excerpt """ # try extracting meta description tag excerpt = '' if try_meta: excerpt = xpath.get(html, '/html/head/meta[@name="description"]/@content') if not excerpt: # remove these tags and then find biggest text block bad_tags = 'hr', 'br', 'script', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'footer' content = common.remove_tags(xpath.get(html, '/html/body', remove=bad_tags)) if content: excerpt = max((len(p.strip()), p) for p in content.splitlines())[1] return common.unescape(excerpt.strip())[:max_chars]
def normalize(self, url, link): """Normalize the link to avoid duplicates >>> cb = CrawlerCallback() >>> cb.normalize('http://example.com', '../abc.html') 'http://example.com/abc.html' >>> cb.normalize('http://example.com', 'abc.html#link') 'http://example.com/abc.html' >>> cb.normalize('http://example.com', 'abc.html?a=1&b=2') 'http://example.com/abc.html?a=1&b=2' """ link, _ = urlparse.urldefrag(link) # remove hash to avoid duplicates link = common.unescape(link) # parse escaped characters such as & link = urlparse.urljoin(url, link) # support relative links while urlparse.urlsplit(link).path.startswith('/..'): # remove invalid parent directory link = link.replace('/..', '', 1) return link
def handle_response(self, request, response): '''handle the response''' if response.status == 200: parser = XmlParser.DynamicParser(response.body) # Retrieve groups for group_dict in parser.groups: group_id = group_dict['groupId'] group_name = group_dict['name'] if group_id in self.session.groups: self.session.groups[group_id].name = group_name else: self.session.groups[group_id] = \ e3.Group(group_name, group_id) # Retrieve contacts for contact_dict in parser.contacts: if 'isMessengerUser' in contact_dict \ and 'passportName' in contact_dict \ and contact_dict['isMessengerUser'] == 'true': # valid email = contact_dict['passportName'].lower() if email in self.session.contacts.contacts: contact = self.session.contacts.contacts[email] else: contact = e3.Contact(email) self.session.contacts.contacts[email] = contact else: continue contact.identifier = contact_dict.get('contactId', '') contact.attrs['CID'] = contact_dict.get('CID', '') contact.groups = [] for guid in contact_dict['groupIds']: contact.groups.append(guid) group = self.session.groups[guid] if contact.account not in group.contacts: group.contacts.append(contact.account) for ann in contact_dict['Annotations']: if ann.get('Name', None) == 'AB.NickName': contact.alias = urllib.unquote(ann['Value']) break if not contact.nick: contact.nick = urllib.unquote(contact_dict.get( 'displayName', contact.account)) contact.attrs['mobile'] = \ contact_dict.get('isMobileIMEnabled', None) == 'true' contact.attrs['space'] = \ contact_dict.get('hasSpace', None) == 'true' log.debug('dynamic finished') self.session.contacts.me.identifier = \ response.body.split('<contactType>Me</contactType>')\ [1].split('</CID>')[0].split('<CID>')[1].strip() # get our nick try: nick = response.body.split('<contactType>Me</contactType>')\ [1].split('</displayName>')[0].split('<displayName>')[1] nick = common.unescape(nick) except IndexError: nick = self.session.contacts.me.account if not self.session.contacts.me.nick or \ self.session.contacts.me != self.session.account.account: self.session.contacts.me.nick = nick if self.on_login: # set our nick self.command_queue.put(Command('PRP', params=('MFN', urllib.quote(nick)))) self.session.add_event(e3.Event.EVENT_NICK_CHANGE_SUCCEED, nick) if not self.started_from_cache: self.command_queue.put(Command('BLP', params=('BL',))) accounts = self.session.contacts.pending.keys() for account in accounts: # account in pending that is already on some other role # (corrupted userlist) if account in self.session.contacts.contacts or account in self.session.contacts.reverse: del self.session.contacts.pending[account] # this line doen't work for accounts on corrupted userlists # RemovePendingContact(self.session, account).start() self.session.add_event(e3.Event.EVENT_CONTACT_LIST_READY) self.session.logger.add_contact_by_group( self.session.contacts.contacts, self.session.groups) if not self.started_from_cache: for adl in self.session.contacts.get_adls(): self.command_queue.put(Command('ADL', payload=adl)) GetProfile(self.session, self.session.contacts.me.identifier).start() else: log.debug('error requestion dynamic items')
def onGetProfile(self, response): #print response.body if response.status[0] != 200 or \ response.body.find('<ExpressionProfile>') == -1: if self.triedCreatingProfile: # we already tried, but failed.. :( self.affinityCache = '' self.rid = '' else: #user doesn't have a roaming profile, try create one. soap.requests.create_profile(self.proxy, self.reRequestProfile) return expression_profile = response.body.split('</ExpressionProfile>')[0].split('<ExpressionProfile>')[1] self.rid = expression_profile.split('</ResourceID>')[0].split('<ResourceID>')[1] try: nick = response.body.split('</DisplayName>')[0].split('<DisplayName>')[1] nick = common.unescape(nick) except: nick = '' try: pm = response.body.split('</PersonalStatus>')[0].split('<PersonalStatus>')[1] pm = common.unescape(pm) except: pm = '' try: dpurl = response.body.split('</PreAuthURL>')[0].split('<PreAuthURL>')[1] except: try: dpurl = response.body.split('</StaticUserTilePublicURL>')[0].split('<StaticUserTilePublicURL>')[1] except: dpurl = "" photo = response.body.split('</Photo>')[0].split('<Photo>')[1] try: self.dpid = photo.split('</ResourceID>')[0].split('<ResourceID>')[1] except: self.dpid = '' try: modified = (photo.split('</DateModified>')[0].split('<DateModified>')[1])[0:19] except: modified = '' newAvatar = True # compare last avatar date and server avatar date, we can't know # the exact time saved on the server, it's something between the # time that we sent the request and the time the response arrived # that's why I compare with seconds of difference (10 is arbitrary) if modified != '': serverDate = modified savedDate = self.config.user['avatarDate'] try: date1 = datetime.datetime(int(serverDate[0:4]), \ int(serverDate[5:7]), int(serverDate[8:10]), \ int(serverDate[11:13]), int(serverDate[14:16]), \ int(serverDate[17:19]),0,None) date2 = datetime.datetime(int(savedDate[0:4]), \ int(savedDate[5:7]), int(savedDate[8:10]), \ int(savedDate[11:13]), int(savedDate[14:16]), \ int(savedDate[17:19]),0,None) delta = 0 if date2 > date1: delta = date2 - date1 else: delta = date1 - date2 newAvatar = (delta.seconds > 10 or delta.seconds < -10) except: newAvatar = True if dpurl != '' and newAvatar: self.config.user['avatarDate'] = modified gobject.idle_add(self.getPicture, dpurl) self.changeNick(nick, initial=True) self.changePersonalMessage(pm) #it seems these are not on the response anymore try: self.affinityCache = response.body.split('</CacheKey>')[0].split('<CacheKey>')[1] except: self.affinityCache = '' try: name = photo.split('</Name>')[0].split('<Name>')[1] except: name = '' self.profile_retrieved = True
def handle_response(self, request, response): '''handle the response''' if response.status == 200: parser = XmlParser.DynamicParser(response.body) # Retrieve groups for group_dict in parser.groups: group_id = group_dict['groupId'] group_name = group_dict['name'] if group_id in self.session.groups: self.session.groups[group_id].name = group_name else: self.session.groups[group_id] = \ e3.Group(group_name, group_id) # Retrieve contacts for contact_dict in parser.contacts: if 'isMessengerUser' in contact_dict \ and 'passportName' in contact_dict \ and contact_dict['isMessengerUser'] == 'true': # valid email = contact_dict['passportName'].lower() if email in self.session.contacts.contacts: contact = self.session.contacts.contacts[email] else: contact = e3.Contact(email) self.session.contacts.contacts[email] = contact else: continue contact.identifier = contact_dict.get('contactId', '') contact.cid = contact_dict.get('CID', '') contact.groups = [] for guid in contact_dict['groupIds']: contact.groups.append(guid) group = self.session.groups[guid] if contact.account not in group.contacts: group.contacts.append(contact.account) for ann in contact_dict['Annotations']: if ann.get('Name', None) == 'AB.NickName': contact.alias = urllib.unquote(ann['Value']) break if not contact.nick: contact.nick = urllib.unquote(contact_dict.get( 'displayName', contact.account)) contact.attrs['mobile'] = \ contact_dict.get('isMobileIMEnabled', None) == 'true' contact.attrs['space'] = \ contact_dict.get('hasSpace', None) == 'true' log.debug('dynamic finished') self.session.contacts.me.identifier = \ response.body.split('<contactType>Me</contactType>')\ [1].split('</CID>')[0].split('<CID>')[1].strip() # get our nick try: nick = response.body.split('<contactType>Me</contactType>')\ [1].split('</displayName>')[0].split('<displayName>')[1] nick = common.unescape(nick) except IndexError: nick = self.session.contacts.me.account if not self.session.contacts.me.nick or \ self.session.contacts.me != self.session.account.account: self.session.contacts.me.nick = nick if self.on_login: # set our nick self.command_queue.put(Command('PRP', params=('MFN', urllib.quote(nick)))) self.session.add_event(e3.Event.EVENT_NICK_CHANGE_SUCCEED, nick) if not self.started_from_cache: self.command_queue.put(Command('BLP', params=('BL',))) accounts = self.session.contacts.pending.keys() for account in accounts: # account in pending that is already on some other role # (corrupted userlist) if account in self.session.contacts.contacts or account in self.session.contacts.reverse: del self.session.contacts.pending[account] # this line doen't work for accounts on corrupted userlists # RemovePendingContact(self.session, account).start() self.session.add_event(e3.Event.EVENT_CONTACT_LIST_READY) self.session.logger.add_contact_by_group( self.session.contacts.contacts, self.session.groups) if not self.started_from_cache: for adl in self.session.contacts.get_adls(): self.command_queue.put(Command('ADL', payload=adl)) GetProfile(self.session, self.session.contacts.me.identifier).start() else: log.debug('error requestion dynamic items')
def get_redirect(self, url, html): """Check for meta redirects and return redirect URL if found """ match = re.compile('<meta[^>]*?url=(.*?)["\']', re.IGNORECASE).search(html) if match: return urlparse.urljoin(url, common.unescape(match.groups()[0].strip()))
def testUnescape(self): """Test that result of unescape equals expected result.""" for input, expected in self.strings_to_unescape: self.assertEqual(unescape(input), expected)