def evaluate():
if os.environ.get('FLASK_ENV') == 'TESTING':
logger.info('Preparing evaluation database...')
db.create_all()
evaluate_clustering()
logger.info('Cleaning up evaluation database...')
db.session.remove()
db.drop_all()
else:
logger.error('This function must be run with FLASK_ENV=TESTING.')
def evaluate():
if os.environ.get('FLASK_ENV') == 'TESTING':
logger.info('Preparing evaluation database...')
db.create_all()
evaluate_clustering()
logger.info('Cleaning up evaluation database...')
db.session.remove()
db.drop_all()
else:
logger.error('This function must be run with FLASK_ENV=TESTING.')
def active():
"""
Get info about currently executing tasks.
"""
try:
active_tasks = celery.control.inspect().active()
if not active_tasks:
logger.info('No active tasks.')
return False
except IOError as e:
logger.error('Error connecting to MQ. Check that it is running.')
return False
logger.info('There are {0} executing tasks.'.format(len(active_tasks)))
return active_tasks
def active():
"""
Get info about currently executing tasks.
"""
try:
active_tasks = celery.control.inspect().active()
if not active_tasks:
logger.info('No active tasks.')
return False
except IOError as e:
logger.error('Error connecting to MQ. Check that it is running.')
return False
logger.info('There are {0} executing tasks.'.format(len(active_tasks)))
return active_tasks
def _expired(url, file):
"""
Determines if the remote file
is newer than the local file.
"""
req = request.Request(url)
try:
resp = request.urlopen(req)
# Server file last modified.
last_mod = resp.headers['Last-Modified']
last_mod_time = time.strptime(last_mod, '%a, %d %b %Y %H:%M:%S %Z')
# Local file last modified.
file_last_mod = os.path.getmtime(file)
file_last_mod_time = time.gmtime(file_last_mod)
return last_mod_time > file_last_mod_time
except request.HTTPError as e:
logger.error('HTTP Error:', e.code, url)
except request.URLError as e:
logger.error('URL Error:', e.reason, url)
def workers():
"""
Get info about currently available Celery workers.
If none are available, or there are issues connecting
to the MQ, returns False.
Returns:
| dict -- dict of available workers.
OR
| bool -- False if no available workers, or cannot connect ot MQ.
"""
try:
# Get info on available workers.
workers = celery.control.inspect().stats()
if not workers:
logger.error('No Celery workers available.')
return False
except IOError as e:
logger.error('Error connecting to MQ. Check that it is running.')
return False
logger.info('There are {0} workers available.'.format(len(workers)))
return workers
def workers():
"""
Get info about currently available Celery workers.
If none are available, or there are issues connecting
to the MQ, returns False.
Returns:
| dict -- dict of available workers.
OR
| bool -- False if no available workers, or cannot connect ot MQ.
"""
try:
# Get info on available workers.
workers = celery.control.inspect().stats()
if not workers:
logger.error('No Celery workers available.')
return False
except IOError as e:
logger.error('Error connecting to MQ. Check that it is running.')
return False
logger.info('There are {0} workers available.'.format(len(workers)))
return workers
def download(url, save_path, filename=None, progress=False):
"""
Downloads a file from the specified URL.
Will resume an existing download if the target
server supports it (responds with the "Accepts-Range" header).
Args:
| url (str) -- url of the file to download
| save_path (str) -- path to the directory to save the file
| progress (bool) -- output progress bar to stdout
"""
# Strip trailing slash, if there is one.
save_path = save_path.rstrip('\/')
if filename is None:
filename = url.split('/').pop()
file = '{0}/{1}'.format(save_path, filename)
existing_size = 0
# If file already exists,
# but there is not a newer file is on the server...
if os.path.exists(file) and not _expired(url, file):
# Append to existing file.
outfile = open(file, 'ab')
# Figure out how many bytes we've got.
existing_size = outfile.tell()
# Setup request for only the remaining bytes.
headers = {'Range': 'bytes={0}-'.format(existing_size)}
req = request.Request(url, headers=headers)
# Otherwise, create a new/overwrite existing file.
else:
# Create/overwrite file.
outfile = open(file, 'wb')
outfile.seek(0)
# Vanilla request.
req = request.Request(url)
try:
# Get response.
resp = request.urlopen(req)
# Get total size of content.
total_size = float(resp.headers['Content-Length'].strip())
# Check if the file has already been downloaded_size.
if total_size == existing_size:
logger.info('File already downloaded.')
return
# Check that the server accepts ranges.
# If it does not, the server will ignore the Range header,
# And we have to start all over again.
if existing_size > 0 and not resp.headers.get('Accept-Ranges', None):
logger.info('Server does not allow resuming of downloads.')
logger.info('Starting from the beginning! :D')
outfile = open(file, 'wb')
outfile.seek(0)
if progress:
progress_bar( (existing_size/total_size) * 100 )
# Pull out the chunks!
for chunk in iter(lambda: resp.read(CHUNK), b''):
# Write the chunk to the file.
outfile.write(chunk)
# Update existing size.
existing_size += len(chunk)
percent_complete = (existing_size/total_size) * 100
# Show progress.
if progress:
progress_bar(percent_complete)
if progress:
sys.stdout.write('\n')
# Return the download's filepath.
return file
except request.HTTPError as e:
logger.error('HTTP Error:', e.code, url)
except request.URLError as e:
logger.error('URL Error:', e.reason, url)
