def get_energies_from_files(self): log.d('Getting energies from files.') energies_dict = {} for folder_name in FOLDER_NAMES: temp_file_list = [ file for file in os.listdir(folder_name) if file.endswith('.log') ] for molecule_name in temp_file_list: text = None temp_dict = {} with open(folder_name + '/' + molecule_name, 'r') as f: text = f.read() if folder_name in ['1-DFTmin_opt', '2-MP2min_sp']: energy_value = round(float(DFT_RE.findall(text)[-1]), 5) temp_dict[folder_name] = energy_value elif folder_name in ['3-M062Xmin_opt', '4-M062Xmax_sp']: energy_value = round(float(M06_RE.findall(text)[-1]), 5) temp_dict[folder_name] = energy_value if molecule_name[:-4] not in energies_dict: energies_dict[molecule_name[:-4]] = [temp_dict] else: energies_dict[molecule_name[:-4]].append(temp_dict) log.i(f'Energies from {len(energies_dict.keys())} files recovered.') # pp.pprint(energies_dict) return energies_dict
def myMain(): x=input('Enter x:') if argv[1] == 'zen': log.level('info') log.d('debug') log.i('info x={}'.format(x)) log.w('warn') log.e('error') log.c('critical') else: if argv[1] == 'basic': # see Python Docs logging.html 16.6.7. LogRecord attributes logging.basicConfig(style='{',format="{asctime:s}:{filename:s}:{lineno:d}:{msg:s}",level='DEBUG') logging.debug('our debug message x={}'.format(x)) logging.warn('our warn message x={}'.format(x)) else: print("If we don't want to type logging.debug() etc, we have to create our own logger lg=logging.getLogger({})".format(__name__)) lg=logging.getLogger(__name__) if argv[1]=='easy': logging.basicConfig(level='INFO') else: lg.setLevel('WARN') # optional: defaults to all levels lg.addHandler(logging.StreamHandler()) lg.handlers[0].setFormatter(logging.Formatter(style='{',fmt="{filename:s}:{funcName:s}():{lineno:d}:{msg:s}")) lg.debug('our debug mesage') lg.info('our info here')
def get_web_page(url): ''' # Let the user know we are trying to download a webpage # Construct a request with a User-Agent header # Send the request and read the webpage from the response # Convert the webpage from bytes to a string, and return it ''' url = clean_link(url) log.i('getting: ' + url) handle = Request(url) handle.add_header('User-Agent', 'SPD/1.0') try: webpage = urlopen(handle).read() except HTTPError as err: log.w('a problem occured when getting ' + url) if err.code == 404: message = 'the requested page could not be found' else: message = 'the server returned a code of ' + str(err.code) log.w(message) webpage = str(err.code) # pass the return code as a string webpage = str(webpage) # convert from bytes to string log.d(len(webpage)) soup = BeautifulSoup(webpage, 'html.parser') return soup
def action_download_submitted_images(): global args # Download all images from the first page user_submitted = get_submitted_page() get_all_images(user_submitted) if args.recursive: # misnomer while True: # Loop until we can't find a next page link log.d('looking for next page...') user_submitted = page_get_next_page(user_submitted) if user_submitted is None: log.d('no next page') break get_all_images(user_submitted)
def get_energies_from_files(self): log.d('Getting energies from files.') energies_dict = {} folder_names = [ folder for folder in next(os.walk('.'))[1] if folder[0].isnumeric() ] folder_names.sort() for folder_name in folder_names: try: temp_file_list = [ file for file in os.listdir(folder_name) if file.endswith('.log') ] except FileNotFoundError: continue for molecule_name in temp_file_list: text = None temp_dict = {} with open(folder_name + '/' + molecule_name, 'r') as f: text = f.read() try: if 'freq' in folder_name.lower(): free_energ = round(float(FRQ_RE.findall(text)[-1]), 8) enthalp = round(float(ENT_RE.findall(text)[-1]), 8) energy_value = 'Free Energy: ' + str( free_energ) + '\nEnthalpy: ' + str(enthalp) temp_dict[folder_name] = energy_value elif len([ method for method in ['dft', 'mp2min', 'm06', 'b3lyp'] if method in folder_name.lower() ]) > 0: energy_value = round(float(DFT_RE.findall(text)[-1]), 8) temp_dict[folder_name] = energy_value if molecule_name[:-4] not in energies_dict: energies_dict[molecule_name[:-4]] = [temp_dict] else: energies_dict[molecule_name[:-4]].append(temp_dict) except IndexError: log.e(f'Error on \'{folder_name+"/"+molecule_name}\'.') temp_dict[folder_name] = 'missing value' log.i(f'Energies from {len(energies_dict.keys())} files recovered.') return energies_dict, folder_names
def page_get_next_page(webpage): ''' # Find the link to the next page, if it exists # If it does, download and return the page # Otherwise, return None explicitly ''' global args urls = webpage.find_all('a') log.d('found %s urls' % len(urls)) log.d(args.next_page_regex.format(args.user_name)) for url in urls: next_page = re.search(args.next_page_regex.format(args.user_name), str(url)) if next_page and not next_page == []: log.d('matches found for %s' % url) log.d('returning early') return get_web_page(next_page.expand(r'\1').replace('amp;', '')) else: log.d('no matches for %s' % url)
def page_get_next_page(webpage): ''' # Find the link to the next page, if it exists # If it does, download and return the page # Otherwise, return None explicitly ''' global args urls = webpage.find_all('a') log.d('found %s urls' % len(urls)) log.d(args.next_page_regex.format(args.user_name)) for url in urls: next_page = re.search( args.next_page_regex.format(args.user_name), str(url)) if next_page and not next_page == []: log.d('matches found for %s' % url) log.d('returning early') return get_web_page(next_page.expand(r'\1').replace('amp;', '')) else: log.d('no matches for %s' % url)
def clean_link(link): global args log.d('cleaning link') old_link = link if not re.match(r'https?://', link): link = 'https://' + link elif 'https://' not in link: link = link.replace('http://', 'https://') if args.gifv_as_gif: link = link.replace('.gifv', '.gif') if args.webm_as_gif: link = link.replace('.webm', '.gif') if '.gfycat.com/' in link: # fix when config link = link.replace('//zippy.', '//giant.') link = link.replace('//fat.', '//giant.') if args.clean_imgur_links: link = link.replace('?1', '') if args.imgur_force_grid: if re.search(r'[a-zA-Z0-9]{7}[bs]\.', link): link = link.replace('b.', '.') link = link.replace('s.', '.') link = link.replace('////', '//') log.d('%s => %s' % (old_link, link)) return link
def download_image(link): ''' # Let the user know we are trying to download an image at the given link # Prepare the command (wget) to download the image # If wget doesn't exist, we raise a FileNotFound error # Otherwise, if we're on Windows, modify the wget command to avoid an issue # with GnuWin wget and SSL # Finally, we run the constructed command to download the image ''' global args # --no-check-certificate is used on windows because GnuWin wget fails to # verify all certificates for some reason link = clean_link(link) log.i('downloading: ' + link) wget_command = [which('wget'), '-b', '-N', '-o', '/dev/null', link] if (which('wget') is None): raise WGetNotFoundException('Could not find wget') elif operating_system() == 'Windows': log.d('on windows, adjusting wget_command') wget_command = [ which('wget'), '-b', '-N', '-o', 'NUL', '--no-check-certificate', link ] try: while call(wget_command) != 0: time.sleep(.05) log.d('call is not 0') log.i('retrying...') except BlockingIOError: time.sleep(.1) log.d('BlockingIOError!') log.w('retrying...') download_image(link)
def download_image(link): ''' # Let the user know we are trying to download an image at the given link # Prepare the command (wget) to download the image # If wget doesn't exist, we raise a FileNotFound error # Otherwise, if we're on Windows, modify the wget command to avoid an issue # with GnuWin wget and SSL # Finally, we run the constructed command to download the image ''' global args # --no-check-certificate is used on windows because GnuWin wget fails to # verify all certificates for some reason link = clean_link(link) log.i('downloading: ' + link) wget_command = [which('wget'), '-b', '-N', '-o', '/dev/null', link] if (which('wget') is None): raise WGetNotFoundException('Could not find wget') elif operating_system() == 'Windows': log.d('on windows, adjusting wget_command') wget_command = [which('wget'), '-b', '-N', '-o', 'NUL', '--no-check-certificate', link] try: while call(wget_command) != 0: time.sleep(.05) log.d('call is not 0') log.i('retrying...') except BlockingIOError: time.sleep(.1) log.d('BlockingIOError!') log.w('retrying...') download_image(link)
def download_image_gallery(link): ''' # Fetch the HTML page at the given link # If it's a gfycat link, alter the url to point at the gif and download it # Otherwise, find all '//i.imgur.com' links and download each one ''' global args if 'gfycat.com/' in link: log.d('page is gfycat') if not re.search(r'\.(gif|webm)', link): link = link.replace('gfycat', 'fat.gfycat') + '.webm' download_image(link) elif 'imgur.com/' in link: webpage = get_web_page(link) if str(webpage) == '404' and re.search(r'layout/grid', link): log.w('grid layout not found, trying again') webpage = get_web_page(link.replace('/layout/grid', '')) log.d('page is imgur gallery (limited to first 20 images!)') urls = webpage.find_all('a') for url in urls: if re.search(args.imgur_gallery_image_regex, url.get('href')): log.d('matched imgur_gallery_image_regex in %s' % url) download_image(url.get('href')) else: log.d('page is imgur grid') urls = webpage.find_all('img') for url in urls: if url.get('data-src'): if re.search(args.imgur_grid_image_regex, url.get('data-src')): log.d('matched data-src in %s, grid image link' % url) download_image(url.get('data-src')) elif url.get('src'): if re.search(args.imgur_gallery_image_regex, url.get('src')): log.d('matched src in %s, gallery image link' % url) download_image(url.get('src')) else: log.d('no src, data-src in %s' % url)
def gjf_settings(self): config_dict = {} file_list = [file for file in os.listdir() if file.endswith('.pdb')] if len(file_list) == 0: log.e( f'\'.pdb\' files not found in directory \'{os.path.basename(os.getcwd())}\'. Aborting...' ) quit() log.i(f'{len(file_list)} files found. Gathering settings...') while True: try: log.d('How many calculations do you wish to perform?') ncalc = int(input()) log.d( 'Do you want to set number of processors used and memory size available? Default is 8 processors and 1gb of memory. (Y/N) ' ) query = input() nproc, mem = 8, 1 if query.lower() == 'y': log.d('How many procesors do you wish to use? ') nproc = int(input()) log.d('How much memory do you want to use? ') mem = float(input()) break except: log.w('You have not provided a valid input, please try again.') config_dict['ncalc'] = ncalc config_dict['nproc'] = nproc config_dict['mem'] = mem config_dict['chk'] = 'chkfile.chk' for calc_number in range(ncalc): type_list = [ 'DFTmin', 'MP2min', 'MP2max', 'M062Xmin', 'M062Xmax', 'Custom' ] opt_list = [ 'Local minimum', 'Transition State', 'Restart from Checkpoint' ] freq_list = ['Complete', 'no Raman'] method_list = ['B3LYP', 'MP2', 'M062X'] basis_list = ['6-31G(d)', '6-311+G(d,p)'] log.d( f'For calculation number {calc_number}, choose a calculation type preset by its number or type \'custom\' for custom settings...' ) for val, name in enumerate(type_list): print(f'{val:15}) {name}') calc_type = input() log.d( f'For calculation number {calc_number}, do you wish optimization (Y/N):' ) query = input() if query.lower() == 'y': log.d(f'Which optimization type do you wish :') for val, name in enumerate(opt_list): print(f'{val:15}) {name}') opt_type = input() log.d( f'For calculation number {calc_number}, do you wish frequency calculations (Y/N):' ) query = input() if query.lower() == 'y': log.d(f'Which frequency calculation do you wish:') for val, name in enumerate(freq_list): print(f'{val:15}) {name}') opt_type = input() if calc_type.lower() == 'custom': log.d( f'For calculation number {calc_number}, select calculation method:' ) for val, name in enumerate(method_list): print(f'{val:15}) {name}') calc_method = input() log.d( f'For calculation number {calc_number}, select basis set:') for val, name in enumerate(basis_list): print(f'{val:15}) {name}') basis_set = input() log.d( f'For calculation number {calc_number}, do you wish to specify a solvent (Y/N):' ) query = input() if query.lower() == 'y': query_text1 = 'Please input solvent name:' solvent = input(f'{query_text1:15}') pp.pprint(config_dict) return config_dict