def evaluate(): if os.environ.get('FLASK_ENV') == 'TESTING': logger.info('Preparing evaluation database...') db.create_all() evaluate_clustering() logger.info('Cleaning up evaluation database...') db.session.remove() db.drop_all() else: logger.error('This function must be run with FLASK_ENV=TESTING.')
def active(): """ Get info about currently executing tasks. """ try: active_tasks = celery.control.inspect().active() if not active_tasks: logger.info('No active tasks.') return False except IOError as e: logger.error('Error connecting to MQ. Check that it is running.') return False logger.info('There are {0} executing tasks.'.format(len(active_tasks))) return active_tasks
def _expired(url, file): """ Determines if the remote file is newer than the local file. """ req = request.Request(url) try: resp = request.urlopen(req) # Server file last modified. last_mod = resp.headers['Last-Modified'] last_mod_time = time.strptime(last_mod, '%a, %d %b %Y %H:%M:%S %Z') # Local file last modified. file_last_mod = os.path.getmtime(file) file_last_mod_time = time.gmtime(file_last_mod) return last_mod_time > file_last_mod_time except request.HTTPError as e: logger.error('HTTP Error:', e.code, url) except request.URLError as e: logger.error('URL Error:', e.reason, url)
def workers(): """ Get info about currently available Celery workers. If none are available, or there are issues connecting to the MQ, returns False. Returns: | dict -- dict of available workers. OR | bool -- False if no available workers, or cannot connect ot MQ. """ try: # Get info on available workers. workers = celery.control.inspect().stats() if not workers: logger.error('No Celery workers available.') return False except IOError as e: logger.error('Error connecting to MQ. Check that it is running.') return False logger.info('There are {0} workers available.'.format(len(workers))) return workers
def download(url, save_path, filename=None, progress=False): """ Downloads a file from the specified URL. Will resume an existing download if the target server supports it (responds with the "Accepts-Range" header). Args: | url (str) -- url of the file to download | save_path (str) -- path to the directory to save the file | progress (bool) -- output progress bar to stdout """ # Strip trailing slash, if there is one. save_path = save_path.rstrip('\/') if filename is None: filename = url.split('/').pop() file = '{0}/{1}'.format(save_path, filename) existing_size = 0 # If file already exists, # but there is not a newer file is on the server... if os.path.exists(file) and not _expired(url, file): # Append to existing file. outfile = open(file, 'ab') # Figure out how many bytes we've got. existing_size = outfile.tell() # Setup request for only the remaining bytes. headers = {'Range': 'bytes={0}-'.format(existing_size)} req = request.Request(url, headers=headers) # Otherwise, create a new/overwrite existing file. else: # Create/overwrite file. outfile = open(file, 'wb') outfile.seek(0) # Vanilla request. req = request.Request(url) try: # Get response. resp = request.urlopen(req) # Get total size of content. total_size = float(resp.headers['Content-Length'].strip()) # Check if the file has already been downloaded_size. if total_size == existing_size: logger.info('File already downloaded.') return # Check that the server accepts ranges. # If it does not, the server will ignore the Range header, # And we have to start all over again. if existing_size > 0 and not resp.headers.get('Accept-Ranges', None): logger.info('Server does not allow resuming of downloads.') logger.info('Starting from the beginning! :D') outfile = open(file, 'wb') outfile.seek(0) if progress: progress_bar( (existing_size/total_size) * 100 ) # Pull out the chunks! for chunk in iter(lambda: resp.read(CHUNK), b''): # Write the chunk to the file. outfile.write(chunk) # Update existing size. existing_size += len(chunk) percent_complete = (existing_size/total_size) * 100 # Show progress. if progress: progress_bar(percent_complete) if progress: sys.stdout.write('\n') # Return the download's filepath. return file except request.HTTPError as e: logger.error('HTTP Error:', e.code, url) except request.URLError as e: logger.error('URL Error:', e.reason, url)