def process_ami(modulus, remainder):
"""
We're running on the AMI instance - so actually do the work
Find the files and move them to S3
:return:
"""
# Connect to the database - the login string is set in the database package
# Start the shutdown signal poller to check when this instance must close
start_poll()
engine = create_engine(DB_LOGIN)
connection = engine.connect()
try:
# Check the processed data
try:
LOG.info('Updating state information')
processed_data(connection, modulus, remainder)
except Exception:
LOG.exception('processed_data(): an exception occurred')
# Store files
try:
LOG.info('Storing files')
store_files(connection, modulus, remainder)
except Exception:
LOG.exception('store_files(): an exception occurred')
# Delete galaxy data - commits happen inside
try:
LOG.info('Deleting galaxy data')
delete_galaxy_data(connection, modulus, remainder)
except Exception:
LOG.exception('delete_galaxy_data(): an exception occurred')
# Delete register data - commits happen inside
try:
LOG.info('Deleting register data')
delete_register_data(connection, modulus, remainder)
except Exception:
LOG.exception('delete_register_data(): an exception occurred')
# Archive to HDF5
try:
LOG.info('Archiving to HDF5')
archive_to_hdf5(connection, modulus, remainder)
except Exception:
LOG.exception('archive_to_hdf5(): an exception occurred')
except SystemExit:
LOG.info(
'Spot Instance Terminate Notice received, archive_task is shutting down'
)
finally:
connection.close()
def process_ami(modulus, remainder):
"""
We're running on the AMI instance - so actually do the work
Find the files and move them to S3
:return:
"""
# Connect to the database - the login string is set in the database package
# Start the shutdown signal poller to check when this instance must close
start_poll()
engine = create_engine(DB_LOGIN)
connection = engine.connect()
try:
# Check the processed data
try:
LOG.info('Updating state information')
processed_data(connection, modulus, remainder)
except Exception:
LOG.exception('processed_data(): an exception occurred')
# Store files
try:
LOG.info('Storing files')
store_files(connection, modulus, remainder)
except Exception:
LOG.exception('store_files(): an exception occurred')
# Delete galaxy data - commits happen inside
try:
LOG.info('Deleting galaxy data')
delete_galaxy_data(connection, modulus, remainder)
except Exception:
LOG.exception('delete_galaxy_data(): an exception occurred')
# Delete register data - commits happen inside
try:
LOG.info('Deleting register data')
delete_register_data(connection, modulus, remainder)
except Exception:
LOG.exception('delete_register_data(): an exception occurred')
# Archive to HDF5
try:
LOG.info('Archiving to HDF5')
archive_to_hdf5(connection, modulus, remainder)
except Exception:
LOG.exception('archive_to_hdf5(): an exception occurred')
except SystemExit:
LOG.info('Spot Instance Terminate Notice received, archive_task is shutting down')
finally:
connection.close()
def process_ami():
"""
We're running on the AMI instance - so actually do the work
Find the files and move them to S3
:return:
"""
# Connect to the database - the login string is set in the database package
ENGINE = create_engine(DB_LOGIN)
connection = ENGINE.connect()
try:
# Check the processed data
try:
processed_data(connection)
except:
LOG.exception('processed_data(): an exception occurred')
# Store files
try:
store_files(connection)
except:
LOG.exception('store_files(): an exception occurred')
# Delete galaxy data - commits happen inside
try:
delete_galaxy_data(connection)
except:
LOG.exception('delete_galaxy_data(): an exception occurred')
# Archive the BOINC stats
try:
archive_boinc_stats()
except:
LOG.exception('archive_boinc_stats(): an exception occurred')
# Archive to HDF5
try:
archive_to_hdf5(connection)
except:
LOG.exception('archive_to_hdf5(): an exception occurred')
finally:
connection.close()
import os
import sys
# Setup the Python Path as we may be running this via ssh
base_path = os.path.dirname(__file__)
sys.path.append(os.path.abspath(os.path.join(base_path, '..')))
sys.path.append(
os.path.abspath(os.path.join(base_path, '../../../../boinc/py')))
import argparse
from utils.logging_helper import config_logger
from archive.store_files_mod import store_files
from utils.readable_dir import ReadableDir
LOG = config_logger(__name__)
LOG.info('PYTHONPATH = {0}'.format(sys.path))
parser = argparse.ArgumentParser('Copy files into S3')
parser.add_argument('-d',
'--dir',
action=ReadableDir,
nargs=1,
help='where the HDF5 files are')
args = vars(parser.parse_args())
DIR = args['dir']
file_count = store_files(DIR)
LOG.info('All Done. Stored %d files.', file_count)
# MA 02111-1307 USA
#
"""
Store the files to NGAS
"""
import os
import sys
# Setup the Python Path as we may be running this via ssh
base_path = os.path.dirname(__file__)
sys.path.append(os.path.abspath(os.path.join(base_path, '../../src')))
sys.path.append(os.path.abspath(os.path.join(base_path, '../../../../boinc/py')))
import argparse
from utils.logging_helper import config_logger
from archive.store_files_mod import store_files
from utils.readable_dir import ReadableDir
LOG = config_logger(__name__)
LOG.info('PYTHONPATH = {0}'.format(sys.path))
parser = argparse.ArgumentParser('Copy files into S3')
parser.add_argument('-d','--dir', action=ReadableDir, nargs=1, help='where the HDF5 files are')
args = vars(parser.parse_args())
DIR = args['dir']
file_count = store_files(DIR)
LOG.info('All Done. Stored %d files.', file_count)
