def get_urls(): """ Runs sequence of searches on NIST website which return all mixtures. Writes list of URLs to resources/urls.txt. """ urls = [ 'http://ilthermo.boulder.nist.gov/ILT2/ilsearch?cmp=&ncmp=1&year=&auth=&keyw=&prp=0', 'http://ilthermo.boulder.nist.gov/ILT2/ilsearch?cmp=&ncmp=2&year=&auth=&keyw=&prp=0', 'http://ilthermo.boulder.nist.gov/ILT2/ilsearch?cmp=&ncmp=3&year=&auth=&keyw=&prp=0' ] nist_spider = NistSpider() data = nist_spider.get_urls(urls) with open('resources/urls.txt', 'w') as f: f.write(json.dumps(data))
def get_listings(): """ Executes request for each individual URL and stores result in resources/data.txt. """ try: with open('resources/urls.txt') as f: urls = json.loads(f.read()) except IOError: print('Call materials.py with --get-urls first') sys.exit(1) nist_spider = NistSpider() data = nist_spider.get_listings(urls) with open('resources/data.txt', 'w') as f: f.write(json.dumps(data))