def parseUBX(self, command, tid, params, payload):
'''this function parses the UBX payload, and sets the personal
message or current media'''
pm = payload.split('<PSM>')[ 1 ].split('</PSM>')[ 0 ]
pm = common.unescape(pm)
self.contactManager.setContactPersonalMessage(tid, pm)
if payload.find('<CurrentMedia>') != -1:
media = payload.split('<CurrentMedia>')[1]\
.split('</CurrentMedia>')[0]
mhead = media.find('\\0Music\\01\\0')
if mhead != -1:
# FIXME: OH GAWD PLEASE GET THIS OUTTA HERE
media = '\xe2\x99\xab ' + media[mhead+12:]
margs = media.split('\\0')
media = margs[0]
for args in range(1, len(margs)):
media = media.replace('{%s}' % (args-1), margs[args])
media = common.unescape(media)
self.contactManager.setContactPersonalMessage(tid, media)
if self.lastCommand != 'ILN':
#TODO: is this signal needed?
self.emit('personal-message-changed', tid, pm)
contact = self.contactManager.getContact(tid)
self.emit('user-attr-changed', contact)
def onGetProfile(self, response):
#print response.body
if response.status[0] != 200 or \
response.body.find('<ExpressionProfile>') == -1:
if self.triedCreatingProfile:
# we already tried, but failed.. :(
self.affinityCache = ''
self.rid = ''
else:
#user doesn't have a roaming profile, try create one.
soap.requests.create_profile(self.proxy, self.reRequestProfile)
return
try:
nick = response.body.split('</DisplayName>')[0].split(
'<DisplayName>')[1]
nick = common.unescape(nick)
except:
nick = ''
try:
pm = response.body.split('</PersonalStatus>')[0].split(
'<PersonalStatus>')[1]
pm = common.unescape(pm)
except:
pm = ''
try:
dpurl = response.body.split('</StaticUserTilePublicURL>')[0].split(
'<StaticUserTilePublicURL>')[1]
photo = response.body.split('</Photo>')[0].split('<Photo>')[1]
self.dpid = photo.split('</ResourceID>')[0].split(
'<ResourceID>')[1]
name = photo.split('</Name>')[0].split('Name>')[1]
except:
dpurl = ''
self.dpid = ''
if dpurl != '' and name != self.config.glob['Id']:
gobject.idle_add(self.getPicture, dpurl)
self.changeNick(nick, initial=True)
self.changePersonalMessage(pm)
try:
self.affinityCache = response.body.split('</CacheKey>')[0].split(
'<CacheKey>')[1]
except:
self.affinityCache = ''
expression_profile = response.body.split(
'</ExpressionProfile>')[0].split('<ExpressionProfile>')[1]
self.rid = expression_profile.split('</ResourceID>')[0].split(
'<ResourceID>')[1]
self.profile_retrieved = True
def getNickFromDynamicItems(xml):
try:
nick = xml.split('<contactType>Me</contactType>')\
[1].split('</displayName>')[0].split('<displayName>')[1]
return common.unescape(nick)
except IndexError:
return ''
def get_redirect(url, html):
"""Check for meta redirects and return redirect URL if found
"""
match = re.compile('<meta[^>]*?url=(.*?)["\']', re.IGNORECASE).search(html)
if match:
return urlparse.urljoin(url,
common.unescape(match.groups()[0].strip()))
def normalize(link):
"""Normalize the link to avoid duplicates
"""
if '#' in link:
# remove internal links to avoid duplicates
link = link[:link.index('#')]
link = common.unescape(link) # remove & from link
return urlparse.urljoin(url, link) # support relative links
def normalize(link):
"""Normalize the link to avoid duplicates
"""
if '#' in link:
# remove internal links to avoid duplicates
link = link[:link.index('#')]
link = common.unescape(link) # remove & from link
return urlparse.urljoin(url, link) # support relative links
def run_command(tokens):
"""Execute a command line (treat internal and external appropriately"""
# Cleanup environment
for var in pseudo_vars:
if var in os.environ.keys():
del os.environ[var]
if tokens[0] == 'exit':
internal_exit('Bye!')
elif tokens[0].lower() == 'cd' and [t for t in tokens if t in sep_tokens
] == []:
# This is a single CD command -- use our custom, more handy CD
internal_cd([unescape(t) for t in tokens[1:]])
else:
if set(sep_tokens).intersection(tokens) == set([]):
# This is a simple (non-compound) command
# Crude hack so that we return to the prompt when starting GUI
# applications: if we think that the first token on the given command
# line is an executable, check its PE header to decide whether it's
# GUI application. If it is, spawn the process and then get on with
# life.
cmd = expand_env_vars(tokens[0].strip('"'))
dir, name = os.path.split(cmd)
ext = os.path.splitext(name)[1]
if not ext or ext in exec_extensions:
# Executable given
app = cmd
else:
# Not an executable -- search for the associated application
if os.path.isfile(cmd):
app = associated_application(ext)
else:
# No application will be spawned if the file doesn't exist
app = None
if app:
executable = full_executable_path(app)
if executable and os.path.splitext(
executable)[1].lower() == '.exe':
# This is an exe file, try to figure out whether it's a GUI
# or console application
if is_gui_application(executable):
import subprocess
s = u' '.join([expand_tilde(t) for t in tokens])
subprocess.Popen(s.encode(sys.getfilesystemencoding()),
shell=True)
return
# Regular (external) command
start_time = time.time()
run_in_cmd(tokens)
console_window = win32console.GetConsoleWindow()
if win32gui.GetForegroundWindow(
) != console_window and time.time() - start_time > 15:
# If the window is inactive, flash after long tasks
win32gui.FlashWindowEx(console_window, win32con.FLASHW_ALL, 3, 750)
def run_command(tokens):
"""Execute a command line (treat internal and external appropriately"""
# Cleanup environment
for var in pseudo_vars:
if var in os.environ.keys():
del os.environ[var]
if tokens[0] == 'exit':
internal_exit('Bye!')
elif is_pure_cd(tokens):
# This is a single CD command -- use our custom, more handy CD
internal_cd([unescape(t) for t in tokens[1:]])
else:
if set(sep_tokens).intersection(tokens) == set([]):
# This is a simple (non-compound) command
# Crude hack so that we return to the prompt when starting GUI
# applications: if we think that the first token on the given command
# line is an executable, check its PE header to decide whether it's
# GUI application. If it is, spawn the process and then get on with
# life.
cmd = expand_env_vars(tokens[0].strip('"'))
dir, name = os.path.split(cmd)
ext = os.path.splitext(name)[1]
if not ext or ext in exec_extensions:
# Executable given
app = cmd
else:
# Not an executable -- search for the associated application
if os.path.isfile(cmd):
app = associated_application(ext)
else:
# No application will be spawned if the file doesn't exist
app = None
if app:
executable = full_executable_path(app)
if executable and os.path.splitext(executable)[1].lower() == '.exe':
# This is an exe file, try to figure out whether it's a GUI
# or console application
if is_gui_application(executable):
import subprocess
s = u' '.join([expand_tilde(t) for t in tokens])
subprocess.Popen(s.encode(sys.getfilesystemencoding()), shell=True)
return
# Regular (external) command
start_time = time.time()
run_in_cmd(tokens)
console_window = win32console.GetConsoleWindow()
if win32gui.GetForegroundWindow() != console_window and time.time() - start_time > 15:
# If the window is inactive, flash after long t1asks
win32gui.FlashWindowEx(console_window, win32con.FLASHW_ALL, 3, 750)
def getNickFromDynamicItems(self, dynamicItems):
# get the nick from the XML
try:
temp = dynamicItems.split('<contactType>Me</contactType>')\
[1].split('</displayName>')[0]
nick = common.unescape(temp.split('<displayName>')[1])
except:
nick = self.user
if nick != self.nick:
oldNick = self.nick
self.nick = nick
self.emit('self-nick-changed', oldNick, self.nick)
def gtrans_get(self, url, **kwargs):
"""Get page via Google Translation
"""
url = 'http://translate.google.com/translate?sl=nl&anno=2&u=%s' % urllib.quote(url)
html = self.get(url, **kwargs)
if html:
m = re.compile(r'<frame src="([^"]+)" name=c>', re.DOTALL|re.IGNORECASE).search(html)
if m:
frame_src = urlparse.urljoin(url, common.unescape(m.groups()[0].strip()))
# force to check redirect here
if kwargs.has_key('num_redirects'): kwargs['num_redirects'] = 1
html = self.get(frame_src, **kwargs)
if html:
# remove google translations content
return re.compile(r'<span class="google-src-text".+?</span>', re.DOTALL|re.IGNORECASE).sub('', html)
def gtrans_get(self, url, **kwargs):
"""Get page via Google Translation
"""
url = 'http://translate.google.com/translate?sl=nl&anno=2&u=%s' % urllib.quote(url)
html = self.get(url, **kwargs)
if html:
m = re.compile(r'<frame src="([^"]+)" name=c>', re.DOTALL|re.IGNORECASE).search(html)
if m:
frame_src = urlparse.urljoin(url, common.unescape(m.groups()[0].strip()))
# force to check redirect here
if kwargs.has_key('num_redirects'): kwargs['num_redirects'] = 1
html = self.get(frame_src, **kwargs)
if html:
# remove google translations content
return re.compile(r'<span class="google-src-text".+?</span>', re.DOTALL|re.IGNORECASE).sub('', html)
def gtrans_get(self, url, **kwargs):
"""Download webpage via Google Translation
"""
url = 'http://translate.google.com/translate?sl=nl&anno=2&u=%s' % urllib.quote(url)
html = self.get(url, **kwargs)
if html:
m = re.compile(r'<iframe[^<>]*src="([^"]+)"[^<>]*name=c', re.DOTALL|re.IGNORECASE).search(html)
if m:
frame_src = urlparse.urljoin(url, common.unescape(m.groups()[0].strip()))
# force to check redirect here
html = self.get(frame_src, **kwargs)
if html:
# remove google translations content
return re.compile(r'<span class="google-src-text".+?</span>', re.DOTALL|re.IGNORECASE).sub('', html)
return self.settings.default
def gtrans_get(self, url, **kwargs):
"""Download webpage via Google Translation
"""
url = 'http://translate.google.com/translate?sl=nl&anno=2&u=%s' % urllib.quote(url)
html = self.get(url, **kwargs)
if html:
m = re.compile(r'<iframe[^<>]*src="([^"]+)"[^<>]*name=c', re.DOTALL|re.IGNORECASE).search(html)
if m:
frame_src = urlparse.urljoin(url, common.unescape(m.groups()[0].strip()))
# force to check redirect here
html = self.get(frame_src, **kwargs)
if html:
# remove google translations content
return re.compile(r'<span class="google-src-text".+?</span>', re.DOTALL|re.IGNORECASE).sub('', html)
return self.settings.default
def get_excerpt(html, try_meta=False, max_chars=255):
"""Extract excerpt from this HTML by finding largest text block
try_meta indicates whether to try extracting from meta description tag
max_chars is the maximum number of characters for the excerpt
"""
# try extracting meta description tag
excerpt = ''
if try_meta:
excerpt = xpath.get(html, '/html/head/meta[@name="description"]/@content')
if not excerpt:
# remove these tags and then find biggest text block
bad_tags = 'hr', 'br', 'script', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'
content = common.remove_tags(xpath.get(html, '/html/body', remove=bad_tags))
if content:
excerpt = max((len(p.strip()), p) for p in content.splitlines())[1]
return common.unescape(excerpt.strip())[:max_chars]
def get_excerpt(html, try_meta=False, max_chars=10000):
"""Extract excerpt from this HTML by finding largest text block
try_meta indicates whether to try extracting from meta description tag
max_chars is the maximum number of characters for the excerpt
"""
# try extracting meta description tag
excerpt = ''
if try_meta:
excerpt = xpath.get(html, '/html/head/meta[@name="description"]/@content')
if not excerpt:
# remove these tags and then find biggest text block
bad_tags = 'hr', 'br', 'script', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'footer'
content = common.remove_tags(xpath.get(html, '/html/body', remove=bad_tags))
if content:
excerpt = max((len(p.strip()), p) for p in content.splitlines())[1]
return common.unescape(excerpt.strip())[:max_chars]
def normalize(self, url, link):
"""Normalize the link to avoid duplicates
>>> cb = CrawlerCallback()
>>> cb.normalize('http://example.com', '../abc.html')
'http://example.com/abc.html'
>>> cb.normalize('http://example.com', 'abc.html#link')
'http://example.com/abc.html'
>>> cb.normalize('http://example.com', 'abc.html?a=1&b=2')
'http://example.com/abc.html?a=1&b=2'
"""
link, _ = urlparse.urldefrag(link) # remove hash to avoid duplicates
link = common.unescape(link) # parse escaped characters such as &
link = urlparse.urljoin(url, link) # support relative links
while urlparse.urlsplit(link).path.startswith('/..'):
# remove invalid parent directory
link = link.replace('/..', '', 1)
return link
def normalize(self, url, link):
"""Normalize the link to avoid duplicates
>>> cb = CrawlerCallback()
>>> cb.normalize('http://example.com', '../abc.html')
'http://example.com/abc.html'
>>> cb.normalize('http://example.com', 'abc.html#link')
'http://example.com/abc.html'
>>> cb.normalize('http://example.com', 'abc.html?a=1&b=2')
'http://example.com/abc.html?a=1&b=2'
"""
link, _ = urlparse.urldefrag(link) # remove hash to avoid duplicates
link = common.unescape(link) # parse escaped characters such as &
link = urlparse.urljoin(url, link) # support relative links
while urlparse.urlsplit(link).path.startswith('/..'):
# remove invalid parent directory
link = link.replace('/..', '', 1)
return link
def handle_response(self, request, response):
'''handle the response'''
if response.status == 200:
parser = XmlParser.DynamicParser(response.body)
# Retrieve groups
for group_dict in parser.groups:
group_id = group_dict['groupId']
group_name = group_dict['name']
if group_id in self.session.groups:
self.session.groups[group_id].name = group_name
else:
self.session.groups[group_id] = \
e3.Group(group_name, group_id)
# Retrieve contacts
for contact_dict in parser.contacts:
if 'isMessengerUser' in contact_dict \
and 'passportName' in contact_dict \
and contact_dict['isMessengerUser'] == 'true':
# valid
email = contact_dict['passportName'].lower()
if email in self.session.contacts.contacts:
contact = self.session.contacts.contacts[email]
else:
contact = e3.Contact(email)
self.session.contacts.contacts[email] = contact
else:
continue
contact.identifier = contact_dict.get('contactId', '')
contact.attrs['CID'] = contact_dict.get('CID', '')
contact.groups = []
for guid in contact_dict['groupIds']:
contact.groups.append(guid)
group = self.session.groups[guid]
if contact.account not in group.contacts:
group.contacts.append(contact.account)
for ann in contact_dict['Annotations']:
if ann.get('Name', None) == 'AB.NickName':
contact.alias = urllib.unquote(ann['Value'])
break
if not contact.nick:
contact.nick = urllib.unquote(contact_dict.get(
'displayName', contact.account))
contact.attrs['mobile'] = \
contact_dict.get('isMobileIMEnabled', None) == 'true'
contact.attrs['space'] = \
contact_dict.get('hasSpace', None) == 'true'
log.debug('dynamic finished')
self.session.contacts.me.identifier = \
response.body.split('<contactType>Me</contactType>')\
[1].split('</CID>')[0].split('<CID>')[1].strip()
# get our nick
try:
nick = response.body.split('<contactType>Me</contactType>')\
[1].split('</displayName>')[0].split('<displayName>')[1]
nick = common.unescape(nick)
except IndexError:
nick = self.session.contacts.me.account
if not self.session.contacts.me.nick or \
self.session.contacts.me != self.session.account.account:
self.session.contacts.me.nick = nick
if self.on_login:
# set our nick
self.command_queue.put(Command('PRP', params=('MFN',
urllib.quote(nick))))
self.session.add_event(e3.Event.EVENT_NICK_CHANGE_SUCCEED, nick)
if not self.started_from_cache:
self.command_queue.put(Command('BLP', params=('BL',)))
accounts = self.session.contacts.pending.keys()
for account in accounts:
# account in pending that is already on some other role
# (corrupted userlist)
if account in self.session.contacts.contacts or account in self.session.contacts.reverse:
del self.session.contacts.pending[account]
# this line doen't work for accounts on corrupted userlists
# RemovePendingContact(self.session, account).start()
self.session.add_event(e3.Event.EVENT_CONTACT_LIST_READY)
self.session.logger.add_contact_by_group(
self.session.contacts.contacts, self.session.groups)
if not self.started_from_cache:
for adl in self.session.contacts.get_adls():
self.command_queue.put(Command('ADL', payload=adl))
GetProfile(self.session, self.session.contacts.me.identifier).start()
else:
log.debug('error requestion dynamic items')
def onGetProfile(self, response):
#print response.body
if response.status[0] != 200 or \
response.body.find('<ExpressionProfile>') == -1:
if self.triedCreatingProfile:
# we already tried, but failed.. :(
self.affinityCache = ''
self.rid = ''
else:
#user doesn't have a roaming profile, try create one.
soap.requests.create_profile(self.proxy, self.reRequestProfile)
return
expression_profile = response.body.split('</ExpressionProfile>')[0].split('<ExpressionProfile>')[1]
self.rid = expression_profile.split('</ResourceID>')[0].split('<ResourceID>')[1]
try:
nick = response.body.split('</DisplayName>')[0].split('<DisplayName>')[1]
nick = common.unescape(nick)
except:
nick = ''
try:
pm = response.body.split('</PersonalStatus>')[0].split('<PersonalStatus>')[1]
pm = common.unescape(pm)
except:
pm = ''
try:
dpurl = response.body.split('</PreAuthURL>')[0].split('<PreAuthURL>')[1]
except:
try:
dpurl = response.body.split('</StaticUserTilePublicURL>')[0].split('<StaticUserTilePublicURL>')[1]
except:
dpurl = ""
photo = response.body.split('</Photo>')[0].split('<Photo>')[1]
try:
self.dpid = photo.split('</ResourceID>')[0].split('<ResourceID>')[1]
except:
self.dpid = ''
try:
modified = (photo.split('</DateModified>')[0].split('<DateModified>')[1])[0:19]
except:
modified = ''
newAvatar = True
# compare last avatar date and server avatar date, we can't know
# the exact time saved on the server, it's something between the
# time that we sent the request and the time the response arrived
# that's why I compare with seconds of difference (10 is arbitrary)
if modified != '':
serverDate = modified
savedDate = self.config.user['avatarDate']
try:
date1 = datetime.datetime(int(serverDate[0:4]), \
int(serverDate[5:7]), int(serverDate[8:10]), \
int(serverDate[11:13]), int(serverDate[14:16]), \
int(serverDate[17:19]),0,None)
date2 = datetime.datetime(int(savedDate[0:4]), \
int(savedDate[5:7]), int(savedDate[8:10]), \
int(savedDate[11:13]), int(savedDate[14:16]), \
int(savedDate[17:19]),0,None)
delta = 0
if date2 > date1:
delta = date2 - date1
else:
delta = date1 - date2
newAvatar = (delta.seconds > 10 or delta.seconds < -10)
except:
newAvatar = True
if dpurl != '' and newAvatar:
self.config.user['avatarDate'] = modified
gobject.idle_add(self.getPicture, dpurl)
self.changeNick(nick, initial=True)
self.changePersonalMessage(pm)
#it seems these are not on the response anymore
try:
self.affinityCache = response.body.split('</CacheKey>')[0].split('<CacheKey>')[1]
except:
self.affinityCache = ''
try:
name = photo.split('</Name>')[0].split('<Name>')[1]
except:
name = ''
self.profile_retrieved = True
def handle_response(self, request, response):
'''handle the response'''
if response.status == 200:
parser = XmlParser.DynamicParser(response.body)
# Retrieve groups
for group_dict in parser.groups:
group_id = group_dict['groupId']
group_name = group_dict['name']
if group_id in self.session.groups:
self.session.groups[group_id].name = group_name
else:
self.session.groups[group_id] = \
e3.Group(group_name, group_id)
# Retrieve contacts
for contact_dict in parser.contacts:
if 'isMessengerUser' in contact_dict \
and 'passportName' in contact_dict \
and contact_dict['isMessengerUser'] == 'true':
# valid
email = contact_dict['passportName'].lower()
if email in self.session.contacts.contacts:
contact = self.session.contacts.contacts[email]
else:
contact = e3.Contact(email)
self.session.contacts.contacts[email] = contact
else:
continue
contact.identifier = contact_dict.get('contactId', '')
contact.cid = contact_dict.get('CID', '')
contact.groups = []
for guid in contact_dict['groupIds']:
contact.groups.append(guid)
group = self.session.groups[guid]
if contact.account not in group.contacts:
group.contacts.append(contact.account)
for ann in contact_dict['Annotations']:
if ann.get('Name', None) == 'AB.NickName':
contact.alias = urllib.unquote(ann['Value'])
break
if not contact.nick:
contact.nick = urllib.unquote(contact_dict.get(
'displayName', contact.account))
contact.attrs['mobile'] = \
contact_dict.get('isMobileIMEnabled', None) == 'true'
contact.attrs['space'] = \
contact_dict.get('hasSpace', None) == 'true'
log.debug('dynamic finished')
self.session.contacts.me.identifier = \
response.body.split('<contactType>Me</contactType>')\
[1].split('</CID>')[0].split('<CID>')[1].strip()
# get our nick
try:
nick = response.body.split('<contactType>Me</contactType>')\
[1].split('</displayName>')[0].split('<displayName>')[1]
nick = common.unescape(nick)
except IndexError:
nick = self.session.contacts.me.account
if not self.session.contacts.me.nick or \
self.session.contacts.me != self.session.account.account:
self.session.contacts.me.nick = nick
if self.on_login:
# set our nick
self.command_queue.put(Command('PRP', params=('MFN',
urllib.quote(nick))))
self.session.add_event(e3.Event.EVENT_NICK_CHANGE_SUCCEED, nick)
if not self.started_from_cache:
self.command_queue.put(Command('BLP', params=('BL',)))
accounts = self.session.contacts.pending.keys()
for account in accounts:
# account in pending that is already on some other role
# (corrupted userlist)
if account in self.session.contacts.contacts or account in self.session.contacts.reverse:
del self.session.contacts.pending[account]
# this line doen't work for accounts on corrupted userlists
# RemovePendingContact(self.session, account).start()
self.session.add_event(e3.Event.EVENT_CONTACT_LIST_READY)
self.session.logger.add_contact_by_group(
self.session.contacts.contacts, self.session.groups)
if not self.started_from_cache:
for adl in self.session.contacts.get_adls():
self.command_queue.put(Command('ADL', payload=adl))
GetProfile(self.session, self.session.contacts.me.identifier).start()
else:
log.debug('error requestion dynamic items')
def get_redirect(self, url, html):
"""Check for meta redirects and return redirect URL if found
"""
match = re.compile('<meta[^>]*?url=(.*?)["\']', re.IGNORECASE).search(html)
if match:
return urlparse.urljoin(url, common.unescape(match.groups()[0].strip()))
def testUnescape(self):
"""Test that result of unescape equals expected result."""
for input, expected in self.strings_to_unescape:
self.assertEqual(unescape(input), expected)
