def parse_ieee_response(response): if not isinstance(response, str): response = response.decode('utf-8') json_response = json.loads(response) articles = json_response['articles'] ieee_data_list = [] for article in articles: url = utils.check_json_key_for_none(article, 'pdf_url') title = utils.check_json_key_for_none(article, 'title') abstract = utils.check_json_key_for_none(article, 'abstract') journal_name = utils.check_json_key_for_none(article, 'publisher') # published_date = utils.check_json_key_for_none(article, 'publication_date') # # if published_date: # published_date = published_date # datetime_object = datetime.strptime('Jun 1 2005 12:00AM', '%b %d %Y %I:%M%p') authors = [author['full_name'] for author in utils.check_json_key_for_none(article['authors'], 'authors')] authors = pap_utils.get_authors_from_list(authors) parsed_api_dict = pap_utils.get_serialized_dict(url=url, display_title=title, title=title, abstract=abstract, journal_name=journal_name, published_date=None, authors=authors, source='IEEE') ieee_data_list.append(parsed_api_dict) return ieee_data_list
def parse_arxiv_response(response): def prepend_ns(tag): return '{http://www.w3.org/2005/Atom}' + tag tree = ET.ElementTree(ET.fromstring(response)) # For XML parsing root_node = tree.getroot() arxiv_data_list = [] for entry in root_node.findall(prepend_ns('entry')): url = entry.find(prepend_ns('link')).attrib['href'] title = utils.check_xml_key_for_none(entry.find(prepend_ns('title'))) abstract = utils.check_xml_key_for_none(entry.find(prepend_ns('summary'))) journal_name = utils.check_xml_key_for_none(entry.find(prepend_ns('arxiv:journal_ref'))) published_date = utils.check_xml_key_for_none(entry.find(prepend_ns('published'))) try: datetime_object = datetime.strptime(published_date, '%Y-%m-%dT%H:%M:%SZ') published_date = datetime_object.strftime('%Y-%m-%d') except: published_date = '' authors_xml = entry.findall(prepend_ns('author')) authours = [author.find(prepend_ns('name')).text for author in authors_xml] authors = pap_utils.get_authors_from_list(authours) parsed_api_dict = pap_utils.get_serialized_dict(url=url, display_title=title, title=title, abstract=abstract, journal_name=journal_name, published_date=published_date, authors=authors, source='arXiv') arxiv_data_list.append(parsed_api_dict) return arxiv_data_list
def parse_plos_response(response): json_response = json.loads(response)['response'] docs = utils.check_json_key_for_none(json_response, 'docs') plos_data_list = [] for doc in docs: url = 'http://journals.plos.org/plosone/article?id=' + utils.check_json_key_for_none(doc, 'id') title = utils.check_json_key_for_none(doc, 'title_display') abstract = utils.check_json_key_for_none(doc, 'abstract')[0] journal_name = utils.check_json_key_for_none(doc, 'journal') published_date = utils.check_json_key_for_none(doc, 'publication_date') try: if published_date: datetime_object = datetime.strptime(published_date, '%Y-%m-%dT%H:%M:%SZ') published_date = datetime_object.strftime('%Y-%m-%d') except: published_date = '' authors = utils.check_json_key_for_none(doc, 'author_display') authors = pap_utils.get_authors_from_list(authors) parsed_api_dict = pap_utils.get_serialized_dict(url=url, display_title=title, title=title, abstract=abstract, journal_name=journal_name, published_date=published_date, authors=authors, source='Plos') plos_data_list.append(parsed_api_dict) return plos_data_list
def parse_springer_data(response): # load the response as JSON json_response = json.loads(response) docs = utils.check_json_key_for_none(json_response, 'records') springer_data_list = [] for doc in docs: url = [ url['value'] for url in utils.check_json_key_for_none(doc, 'url') ] title = utils.check_json_key_for_none(doc, 'title') abstract = utils.check_json_key_for_none(doc, 'abstract') published_date = utils.check_json_key_for_none(doc, 'publicationDate') authors = [ creator['creator'] for creator in utils.check_json_key_for_none(doc, 'creators') ] authors = pap_utils.get_authors_from_list(authors) parsed_api_dict = pap_utils.get_serialized_dict( url=url[0], display_title=title, title=title, abstract=abstract, journal_name='', published_date=published_date, authors=authors, source='Springer') springer_data_list.append(parsed_api_dict) return springer_data_list