def test_download_from_dict(self): with tempfile.TemporaryDirectory() as temp_dir: arxiv.download(self.paper_dict, dirpath=temp_dir) self.assertTrue( os.path.exists( os.path.join(temp_dir, '1605.08386v1.The_Paper_Title.pdf')))
def download(abstract_url: str, download_dir: str) -> None: paper = get_paper(abstract_url) print(f"saving '{paper.get('title')}'") arxiv.download(paper, dirpath=download_dir, slugify=lambda paper: paper.get("title")) print(f"saved to {download_dir}")
def test_download_with_custom_slugify_from_dict(self): with tempfile.TemporaryDirectory() as temp_dir: arxiv.download(self.paper_dict, slugify=custom_slugify, dirpath=temp_dir) self.assertTrue( os.path.exists(os.path.join(temp_dir, '1605.08386v1.pdf')))
def test_download_with_custom_slugify_from_dict(self): arxiv.download(self.paper_dict, slugify=custom_slugify, dirpath=self.temp_dir) self.assertTrue( os.path.exists( os.path.join(self.temp_dir, '1605.08386v1.pdf') ) )
def _download_papers(self, folder_path): """ Method for downloading retrieved papers. """ for paper in self.fav_papers: arxiv.download(paper, dirpath=folder_path, slugify=self._custom_slugify)
def get_information_arxiv(q_key, f_max_results): keys = arxiv.query( search_query=q_key, max_results=f_max_results, ) q_key = q_key.replace(" ", "_") if not os.path.exists(q_key): os.makedirs(q_key) liste_link = [] for i in range(len(keys)): # print(keys[i]["authors"]) # print(keys[i]["pdf_url"]) # print(keys[i]["title"]) # print("----------") liste_link.append(keys[i]["pdf_url"]) try: arxiv.download(keys[i]) shutil.move(keys[i]["title"] + ".pdf", q_key) except: pass os.remove(keys[i]["title"] + ".pdf")
def test_download_from_query(self): arxiv.download(self.paper_query, dirpath=self.temp_dir) self.assertTrue( os.path.exists( os.path.join( self.temp_dir, '1605.08386v1.Heat_bath_random_walks_with_Markov_bases.pdf' )))
def test_download_tarfile_from_dict(self): arxiv.download(self.paper_dict, dirpath=self.temp_dir, prefer_source_tarfile=True) self.assertTrue( os.path.exists( os.path.join(self.temp_dir, '1605.08386v1.The_Paper_Title.tar.gz')))
def test_download_from_dict(self): arxiv.download(self.paper_dict, dirpath=self.temp_dir) self.assertTrue( os.path.exists( os.path.join( self.temp_dir, '1605.08386v1.The_Paper_Title.pdf') ) )
def test_download_tarfile_from_query(self): arxiv.download(self.paper_query, dirpath=self.temp_dir, prefer_source_tarfile=True) self.assertTrue( os.path.exists( os.path.join( self.temp_dir, '1605.08386v1.Heat_bath_random_walks_with_Markov_bases.tar.gz' )))
def test_download_from_query(self): arxiv.download(self.paper_query, max_results=1, save_path=self.temp_dir) self.assertTrue( os.path.exists( os.path.join( self.temp_dir, 'The_Multi_Agent_Reinforcement_Learning_in_MalmĂ–_MARLĂ–_Competition.pdf' )))
def test_download_from_query(self): with tempfile.TemporaryDirectory() as temp_dir: arxiv.download(self.paper_query, dirpath=temp_dir) self.assertTrue( os.path.exists( os.path.join( temp_dir, '1605.08386v1.Heat_bath_random_walks_with_Markov_bases.pdf' )))
def main(argv): try: opts, args = getopt.getopt(argv, "q:l:o", ["query=", "limit=", "output="]) except: print 'Use syntax: python arxiv-downloader.py -q "network node centrality" -l 50 -o ./input' sys.exit(2) # Init defaults limit = 10 query = False outputDirectory = "./input" # Parse arguments for opt, arg in opts: if opt == "-q": query = arg elif opt == "-h": print 'Use syntax: python arxiv-downloader.py -q "network node centrality" -l 50 -o ./input' elif opt == "l": limit = arg elif opt == "o": outputDirectory = arg if not(query): print 'Use syntax: python arxiv-downloader.py -q "network node centrality" -l 50 -o ./input' sys.exit(2) if not(os.path.isdir(outputDirectory)): print "Given output directory is not a directory." sys.exit(2) else: os.chdir(outputDirectory) res = arxiv.query(query, prune=True, start=0, max_results=limit) i = 0 results = [] # Display titles for elem in res: for key in elem: if key == "title": i += 1 results.append(elem) print str(i) + ". " + elem[key] + "\n" toDownload = input("Enter the numbers of the papers you want to download separated by commas: \n") try: iterator = iter(toDownload) except TypeError: arxiv.download(results[toDownload]) else: for elem in toDownload: arxiv.download(results[elem])
def download_pdf(): """Download papers listed in meta_data folder""" df = pd.read_csv('meta_data/papers.csv', usecols=['id', 'pdf_url', 'title']) for i, (id, title, pdf_url) in enumerate(zip(df.id, df.title, df.pdf_url)): if i % 100 == 0: print(i) try: arxiv.download({'pdf_url': pdf_url, 'title': title}) move(title + '.pdf', str(Path('papers', id + '.pdf'))) except Exception as e: print(e) time.sleep(2.5)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--dir', type=str, default='.', help='directory name') parser.add_argument('--id', type=str, required=True, help='arXiv id') parser.add_argument('--prepend', action='store_true', help='prepend paper id') parser.add_argument('--slugify', action='store_true', help='slugify file name') args = parser.parse_args() paper_obj = arxiv.query(id_list=[args.id])[0] arxiv.download(paper_obj, args.dir, args.prepend, args.slugify)
def pickle2pdf(target_category='cs.LG'): #flag = 'This article has been withdrawn' filelist = [] for year in range(2012,2020): file = os.path.join(os.getcwd(),'data/arxiv/{}_papers.pkl'.format(year)) filelist.append(file) print('filelist ready') num_suc = 0 ; num_fail = 0 global c c = '' for f_i, file in enumerate(filelist): papers = loadpk(file) print('pickled paper loaded {}'.format(f_i)) def custom_slugify(obj): name = obj.get('id').split('/')[-1] #time = obj.get('published').split('T') res = 'data/arxiv/{}/pdf/'.format(c) + name#+ time + '_' + name return res#obj.get('id').split('/')[-1] for paper in papers: arxiv_id = paper['arxivid'] category = paper['categories'] if 'cs.LG' in category: if 'cs.CL' in category: c = 'LG_CL' else: c = 'LG' elif 'cs.CL' in category: c = 'CL' else: pass try: d_paper = ax.query(id_list=[arxiv_id])[0] ax.download(d_paper, slugify=custom_slugify) # res = 'data/arxiv/{}/'.format(target_category) + d_paper.get('id').split('/')[-1] # all_pdf_path.append(res) print('download {} {} succeed.'.format(arxiv_id, c)) with open('suc_ids.txt', 'a') as w: w.write('{}\t{}\n'.format(arxiv_id,c)) w.close() num_suc += 1 except: print('----------download {} {} failed'.format(arxiv_id,c)) with open('faild_ids.txt', 'a') as w: w.write('{}\t{}\n'.format(arxiv_id,c)) w.close() num_fail += 1 print('num_suc: {} , num_fail: {}'.format(num_suc, num_fail)) return
def paper_download(self): # paper_name = self.get_paper_name() self.make_download_directory() author_dir_paths = self.make_author_directory() print("author_dir_paths", author_dir_paths) for author_dir_path in author_dir_paths: try: arxiv.download(obj=self.paper, dirpath=f'{author_dir_path}', slugify=self.get_paper_name) print("author_dir_path:", author_dir_path) time.sleep(3) except Exception as e: logging.warning(f"download_error {e}") time.sleep(3)
def download_from_arxiv(title, my_api_key, my_cse_id, dirname='./'): """Download arxiv paper from the title Args: title (str): Full title of the paper dirname (str): Output directory of the pdf after downloaded Returns: Filename of the downloaded pdf """ title = '"' + title.replace('-', ' ') + '"' # Put quote for exact search title = remove_non_ascii(title) title_prepended = 'ti:' + title results = arxiv.query(title_prepended) logging.debug(results) outfile = get_filename(dirname, title) if not os.path.isfile(outfile): if paper_available_on_arxiv(results): logging.info("Downloading from arxiv: " + title) return arxiv.download( results[0], dirname=dirname, slugify=True ) # When slugify is True, the paper title will be stripped of non-alphanumeric characters before being used as a filename. else: google_query = title + ' filetype:PDF' logging.info("Paper is not on arxiv, downloading from google: " + google_query) download_from_google(google_query, my_api_key, my_cse_id, outfile) else: logging.info( "Paper has already been downloaded previously. Will skip downloading this file (%s)", title) return None
def create_clinks_set(item, dirpath): paper = {'pdf_url': item['pdf_url'], "title": item['title']} summary = item['summary'] title = item['title'] summary = title + '. ' + summary keywords = rake.apply(summary) path = arxiv.download(paper, dirpath=dirpath) cid_to = client.add(path)['Hash'] cid_from_list = [] for keyword in keywords: if keyword[1] > RANK: temp = client.add_str(keyword[0]) cid_from_list.append(temp) else: pass cid_from_list.append(client.add_str(title)) data = {'title': title, 'cid_from': cid_from_list, 'cid_to': cid_to} clinks_df = pd.DataFrame.from_dict(data) return clinks_df
def download_source(eprint='2012.06888', bib=False, dat=False): """Download a tar file from arXiv and choose the right file.""" filename = eprint.replace('/', '-') paper = arxiv.query(id_list=[eprint])[0] paper = arxiv.download(paper, prefer_source_tarfile=True) this_tarfile = tarfile.open(paper, 'r') tarfiles = {} file_count = 0 for this_file in this_tarfile.getnames(): file_type_regex = re.compile(r'^.*(tex|xml|txt)$') if bib: file_type_regex = re.compile(r'^.*(tex|xml|txt|bib|bbl|inc)$') if dat: file_type_regex = re.compile(r'^.*(tex|xml|txt|dat)$') if file_type_regex.match(this_file) or VERBOSE: file_count += 1 tarfiles[file_count] = this_file print(file_count, tarfiles[file_count]) if file_count == 1: file_choice = file_count else: file_choice = input('Choose a file: ') file_choice = int(file_choice) source_file = this_tarfile.extractfile(tarfiles[file_choice]) file_type = re.match(r'.*(\w{3})$', tarfiles[file_choice]).group(1) output = open(filename + '.' + file_type, 'wb') output.write(source_file.read()) output.close() os.unlink(paper) return file_type
def test_download_on_sort(self): asc_papers = download(query='rnn', max_results=3, sort_by='submittedDate', sort_order='ascending') desc_papers = download(query='rnn', max_results=3, sort_by='submittedDate', sort_order='descending') self.assertEqual(type(asc_papers), list) self.assertEqual(len(asc_papers), 3) self.assertEqual(type(desc_papers), list) self.assertEqual(len(desc_papers), 3) for i, paper in enumerate(asc_papers): self.assertIn('title', paper) self.assertIn('abstract', paper) self.assertIn('authors', paper) self.assertIn('publication_time', paper) self.assertIn('arxiv_url', paper) self.assertIn('pdf_url', paper) self.assertIn('journal_reference', paper) if i > 0: self.assertGreaterEqual(paper['publication_time'], asc_papers[i - 1]['publication_time']) for i, paper in enumerate(desc_papers): self.assertIn('title', paper) self.assertIn('abstract', paper) self.assertIn('authors', paper) self.assertIn('publication_time', paper) self.assertIn('arxiv_url', paper) self.assertIn('pdf_url', paper) self.assertIn('journal_reference', paper) if i > 0: self.assertGreaterEqual(desc_papers[i - 1]['publication_time'], paper['publication_time'])
def pickle2pdf(target_category='cs.LG'): #flag = 'This article has been withdrawn' num_withdraw = 0 num_abstract = 0 filelist = [] all_pdf_path = [] for year in range(2012, 2013): file = os.path.join(os.getcwd(), 'data/arxiv/{}_papers.pkl'.format(year)) filelist.append(file) print('filelist ready') data_path = os.path.join(os.getcwd(), 'data/arxiv') w = os.path.join(data_path, 'train_{}.txt'.format(target_category)) log = os.path.join(data_path, 'log.txt') n = 0 for f_i, file in enumerate(filelist): per_num_withdraw = 0 per_num_abstract = 0 papers = loadpk(file) print('pickled paper loaded {}'.format(f_i)) def custom_slugify(obj): name = obj.get('id').split('/')[-1] #time = obj.get('published').split('T') res = 'data/arxiv/{}/pdf/'.format( target_category) + name #+ time + '_' + name print(res) return res #obj.get('id').split('/')[-1] for paper in papers: if n > 10: break arxiv_id = paper['arxivid'] try: d_paper = ax.query(id_list=[arxiv_id])[0] ax.download(d_paper, slugify=custom_slugify) # res = 'data/arxiv/{}/'.format(target_category) + d_paper.get('id').split('/')[-1] # all_pdf_path.append(res) except: print('download {} failed'.format(arxiv_id)) n += 1 return
def download(self, slugify, reload_download_list=False): if reload_download_list: self.prepare_for_download() if not self.to_download: print('Download list is empty. Trying to reload.') self.prepare_for_download() self.download() else: print('Download started...') for elem in tqdm.tqdm(self.to_download): try: arxiv.download(elem, slugify=slugify, dirpath=self.download_path) except: print( 'Something went wrong during downloading {0} by url {1}. Passed' .format(elem['title'], elem['pdf_url'])) print('Download finished!')
def retrieve(self): results = iter(self._results) if self.keep: save_dir = Path.cwd() else: save_dir = None with TemporaryDirectory(dir=save_dir) as temp_dir: for result in results: filename = arxiv.download(result, dirpath=temp_dir) yield self.read_pdf(filename)
def download(self): """ Changes to paper directory. Downlaods pdfs for each paper. """ if not os.path.exists('./papers'): os.mkdir('./papers') os.chdir('./papers') base_url = 'https://arxiv.org/pdf/' for link in self.pdf_links: print('Downloading: {}'.format(link)) paper = { 'pdf_url': base_url + link, 'title': link.split('.pdf')[0] } try: arxiv.download(paper) except: print('Download: {} failed unexpectedly.'.format(link)) pass
def test_download_on_query(self): papers = download(query='rnn', max_results=2) self.assertEqual(type(papers), list) self.assertEqual(len(papers), 2) for paper in papers: self.assertIn('title', paper) self.assertIn('abstract', paper) self.assertIn('authors', paper) self.assertIn('publication_time', paper) self.assertIn('arxiv_url', paper) self.assertIn('pdf_url', paper) self.assertIn('journal_reference', paper)
def getDocumentTextFile(self, document): document_pdf_file_path = arxiv.download(document, DATA_DIRECTORY_PATH) try: document_text = textract.process(document_pdf_file_path, encoding="utf-8") except: #print "\t\t (Conversion to text failed)" os.remove(document_pdf_file_path) else: os.remove(document_pdf_file_path) document_text_file_path = DATA_DIRECTORY_PATH + document_pdf_file_path[:-len( "pdf")] + "txt" document_text_file = open(document_text_file_path, "w") document_text_file.write(document_text) document_text_file.close() return document_text_file_path
def download_arxiv_paper(arxiv_id, feel_lucky: bool = True): """ Input: arxiv_id Return: the downloaded paper folder. """ if not os.path.exists(paper_download_dir): os.makedirs(paper_download_dir) # Don't download again if exists search_this_paper = glob.glob(f'{paper_download_dir}/{arxiv_id}*/') if len(search_this_paper) > 0: return search_this_paper[0] tarpath = arxiv.download({'pdf_url': f"http://arxiv.org/pdf/{arxiv_id}"}, dirpath=paper_download_dir, prefer_source_tarfile=True) with tarfile.open(tarpath, 'r') as tar: tar.extractall(path=tarpath.replace('.tar.gz', '')) return tarpath.replace('.tar.gz', '')
def test_download_on_filters(self): search = 'Multi-Agent Reinforcement Learning' author = 'devlin' journal = 'nips' query = construct_query(search, author, journal) before = "2019-10-12" after = "2018-10-11" _before = datetime.strptime(before, "%Y-%m-%d") _after = datetime.strptime(after, "%Y-%m-%d") papers = download(query=query, max_results=1, before=_before, after=_after) self.assertEqual(type(papers), list) self.assertEqual(len(papers), 1) for paper in papers: self.assertIn('title', paper) self.assertIn('abstract', paper) self.assertIn('authors', paper) self.assertIn('publication_time', paper) self.assertIn('arxiv_url', paper) self.assertIn('pdf_url', paper) self.assertIn('journal_reference', paper) self.assertTrue(search in paper['title'] or search in paper['abstract']) self.assertTrue( any([author.lower() in a.lower() for a in paper['authors']])) self.assertIn(journal.lower(), paper['journal_reference'].lower()) self.assertGreaterEqual(before, paper['publication_time']) self.assertGreaterEqual(paper['publication_time'], after)
def loadArticles(self, inpString, directory, count=10): newDirFl = True try: os.mkdir(directory) except FileExistsError: newDirFl = False articles = arxiv.query(query=inpString, max_results=count * 2) indexFile = open(os.path.join(directory, "index.txt"), 'a') for article in articles: try: filename = arxiv.download(article, dirpath=directory) try: pdf = PyPDF2.PdfFileReader(filename) count -= 1 indexFile.write( os.path.basename(article.id) + ' ' + os.path.basename(filename) + ' 0\n') except Exception: os.remove(filename) except FileExistsError: pass if count == 0: break indexFile.close()
# coding: utf-8 # In[1]: import arxiv import argparse import os from tqdm import tqdm # In[6]: if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('-l', '--list', required=True) parser.add_argument('-o', '--output_dir', default='./') args = parser.parse_args() if os.path.exists(args.output_dir) == False: os.makedirs(args.output_dir) l = open(args.list).readlines() for i in tqdm(range(len(l))): ti = l[i] q = 'ti:"{0}"'.format(ti.strip()).replace('-', ' ') print q r = arxiv.query(q) try: d = arxiv.download(dirname=args.output_dir, obj=r[0]) except Exception as e: print e print 'failed downloading: ', ti
def get_arxiv_link(bot, msg): results = arxiv.query(msg['text'].replace("/arxiv ", ""), max_results=1) bot.sender.sendMessage("Title: {}\nAuthor: {}".format(results[0]['title'], results[0]['author'])) bot.sender.sendDocument(open(arxiv.download(results[0]), "rb"))