def __init__(self, url):

"""

Create BibTex instance from ADS BibTex URL

"""

bibtex = urllib.urlopen(url).readlines()

bibtex = ' '.join([l.strip() for l in bibtex]).strip()

bibtex = bibtex[re.search('@[A-Z]+{', bibtex).start():]

self.type, self.bibcode, self.info = self.parsebib(bibtex)

def __str__(self):

return ','.join(['@' + self.type + '{' + self.bibcode] + ['%s=%s' % (i, j) for i, j in self.info.items()]) + '}'

def parsebib(self, bibtex):

"""

Parse bibtex code into dictionary

"""

r = re.search('(?<=^@)(?P<type>[A-Z]+){(?P<bibcode>\S+)(?P<info>,.+)}$', bibtex)

s = re.split('(,\s\w+\s=\s)', r.group('info'))

info = dict([(i[1:].replace('=', '').strip(), j.strip()) for i, j in zip(s[1::2], s[2::2])])

def __init__(self, *args, **kwargs):

HTMLParser.__init__(self)

self.links = {}

self.tag = ''

self.get_abs = False

self.entities = {}

self.bibtex = None

self.abstract = None

self.title = ''

self.author = []

def mathml(self):

"""

Generate dictionary with MathML -> unicode conversion from

http://www.w3.org/Math/characters/byalpha.html

"""

w3 = 'http://www.w3.org/Math/characters/byalpha.html'

mathml = re.search('(?<=\<pre\>).+(?=</pre>)', urllib.urlopen(w3).read(), re.DOTALL).group()

entities = {}

for l in mathml[:-1].split('\n'):

s = l.split(',')

#ignore double hex values like 'U02266-00338'

if not '-' in s[1]:

#hexadecimal -> int values, for unichr

entities[s[0].strip()] = int(s[1].strip()[1:], 16)

return entities

def parse(self, url):

"""

Feed url into our own HTMLParser and parse found bibtex

"""

self.feed(urllib.urlopen(url).read())

#parse bibtex

if 'bibtex' in self.links:

self.bibtex = BibTex(self.links['bibtex'])

self.title = re.search('(?<={).+(?=})', self.bibtex.info['title']).group().replace('{','').replace('}','')

self.author = [a.strip() for a in

re.search('(?<={).+(?=})', self.bibtex.info['author']).group().split(' and ')]

def handle_starttag(self, tag, attrs):

#abstract

if tag.lower() == 'hr' and self.get_abs:

self.abstract = self.tag.strip().decode('utf-8')

self.get_abs = False

self.tag = ''

#handle old scanned articles abstracts

elif tag.lower() == 'img' and self.get_abs:

self.tag += dict(attrs)['src']

#links

elif tag.lower() == 'a':

if 'href' in dict(attrs):

href = dict(attrs)['href']

query = cgi.parse_qs(urlparse.urlparse(href).query)

if 'bibcode' in query:

if 'link_type' in query:

self.links[query['link_type'][0].lower()] = href

elif 'data_type' in query:

self.links[query['data_type'][0].lower()] = href

def handle_data(self, data):

if self.get_abs:

self.tag += data.replace('\n', ' ')

#beginning of abstract found

if data.strip() == 'Abstract':

self.get_abs = True

#handle html entities

def handle_entityref(self, name):

if self.get_abs:

if name in name2codepoint:

c = name2codepoint[name]

self.tag += unichr(c).encode('utf-8')

else:

#fetch mathml

if not self.entities:

#cache dict

self.entities = self.mathml()

if name in self.entities:

c = self.entities[name]

self.tag += unichr(c).encode('utf-8')

else:

#nothing worked, leave it as-is

self.tag += '&' + name + ';'

#handle unicode chars in utf-8

def handle_charref(self, name):

if self.get_abs:

self.tag += unichr(int(name)).encode('utf-8')

def getPDF(self):

"""

Fetch PDF and save it locally in a temporary file.

Tries by order:

- refereed article

- refereed article using another machine (set ssh_user & ssh_server)

- arXiv preprint

- electronic journal link

"""

if not self.links:

return 'failed'

def filetype(filename):

return subprocess.Popen('file %s' % filename, shell=True,

stdout=subprocess.PIPE,

stderr=subprocess.PIPE).stdout.read()

#refereed

if 'article' in self.links:

url = self.links['article']

#try locally

pdf = tempfile.mktemp() + '.pdf'

urllib.urlretrieve(url, pdf)

if 'PDF document' in filetype(pdf):

return pdf

#try in remote server

# you need to set SSH public key authentication

# for this to work!

elif ssh_user is not None:

pdf = tempfile.mktemp() + '.pdf'

cmd = 'ssh %s@%s \"touch toto.pdf; wget -O toto.pdf \\"%s\\"\"' % (ssh_user, ssh_server, url)

cmd2 = 'scp -q %s@%s:toto.pdf %s' % (ssh_user, ssh_server, pdf)

subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()

subprocess.Popen(cmd2, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()

if 'PDF document' in filetype(pdf):

return pdf

#arXiv

if 'preprint' in self.links:

#arXiv page

url = self.links['preprint']

for line in urllib.urlopen(url).readlines():

if 'dc:identifier' in line:

begin = re.search('dc:identifier="', line).end()

url = line[begin:-2].replace('&#38;', '&').lower()

#get arXiv PDF

pdf = tempfile.mktemp() + '.pdf'

urllib.urlretrieve(url.replace('abs', 'pdf'), pdf)

if 'PDF document' in filetype(pdf):

return pdf

else:

return url

#electronic journal

if 'ejournal' in self.links:

return self.links['ejournal']