def make_request(connection, email_address, galaxy_ids, features, layers, pixel_types): """ Makes a request in the database for the provided galaxies. :param connection: The database connection :param email_address: The email address requesting the galaxies :param galaxy_ids: The database IDs of the galaxies :param features: The features of the galaxies to request :param layers: The layers of the galaxies to request :param pixel_types: The pixel types of the galaxies to request :return: """ # Make some new DB entries for these galaxies transaction = connection.begin() try: LOG.info("Making HDF5_REQUEST entry...") result = connection.execute(HDF5_REQUEST.insert(), profile_id=0, email=email_address, created_at=datetime.datetime.utcnow()) LOG.info("Making features, layers, pixel types entries...") insert_features_layers_pixel_types_db_ids(connection, result.inserted_primary_key, features, layers, pixel_types) LOG.info("Making galaxy entries...") for galaxy in galaxy_ids: # We already checked to ensure these are valid, so throw em in. connection.execute(HDF5_REQUEST_GALAXY.insert(), hdf5_request_id=result.inserted_primary_key, galaxy_id=galaxy) transaction.commit() except: transaction.rollback() raise
sys.path.append(os.path.abspath(os.path.join(base_path, '..'))) import datetime from config import DB_LOGIN from database.database_support_core import HDF5_REQUEST, HDF5_REQUEST_GALAXY from sqlalchemy import create_engine, select from hdf5_to_fits.hdf5_to_fits_mod import generate_files, get_features_layers_galaxies from utils.logging_helper import config_logger LOG = config_logger(__name__) LOG.info('PYTHONPATH = {0}'.format(sys.path)) # First check the galaxy exists in the database engine = create_engine(DB_LOGIN) connection = engine.connect() for request in connection.execute(select([HDF5_REQUEST], distinct=True, from_obj=HDF5_REQUEST.join(HDF5_REQUEST_GALAXY)).where(HDF5_REQUEST_GALAXY.c.state == 0)): # Mark the request as being processed request_id = request[HDF5_REQUEST.c.hdf5_request_id] features, layers, hdf5_request_galaxy_ids = get_features_layers_galaxies(connection, request_id) if len(features) > 0 and len(layers) > 0 and len(hdf5_request_galaxy_ids) > 0: generate_files(connection=connection, hdf5_request_galaxy_ids=hdf5_request_galaxy_ids, email=request[HDF5_REQUEST.c.email], features=features, layers=layers) connection.execute(HDF5_REQUEST.update().where(HDF5_REQUEST.c.hdf5_request_id == request_id).values(updated_at=datetime.datetime.now())) LOG.info('All done') connection.close()
from hdf5_to_fits.hdf5_to_fits_mod import generate_files, get_features_layers_galaxies_pixeltypes from utils.logging_helper import config_logger LOG = config_logger(__name__) LOG.info('Starting HDF5 to FITS') LOG.info('PYTHONPATH = {0}'.format(sys.path)) # First check the galaxy exists in the database engine = create_engine(DB_LOGIN) connection = engine.connect() for request in connection.execute( select( [HDF5_REQUEST], distinct=True, from_obj=HDF5_REQUEST.join(HDF5_REQUEST_GALAXY)).where(HDF5_REQUEST_GALAXY.c.state == 0) ): # Mark the request as being processed request_id = request[HDF5_REQUEST.c.hdf5_request_id] features, layers, hdf5_request_galaxy_ids, pixel_types = get_features_layers_galaxies_pixeltypes(connection, request_id) if len(features) > 0 and len(layers) > 0 and len(hdf5_request_galaxy_ids) > 0: LOG.info('Processing request from profile id: {0}, request made at {1}'.format(request[HDF5_REQUEST.c.profile_id], request[HDF5_REQUEST.c.created_at])) LOG.info('{0} features, {1} layers {2} galaxies'.format(len(features), len(layers), len(hdf5_request_galaxy_ids))) generate_files( connection=connection, hdf5_request_galaxy_ids=hdf5_request_galaxy_ids, email=request[HDF5_REQUEST.c.email], features=features, layers=layers, pixel_types=pixel_types )
features, layers, pixel_types = get_features_and_layers_pixeltypes_cmd_line( args) if len(features) == 0 or len(layers) == 0: parser.print_help() exit(1) # Create a new request # Need to make: # HDF5_request # HDF5_request_feature # HDF5_request_galaxy # HDF5_request_layer # HDF5_request_pixel transaction = connection.begin() result = connection.execute(HDF5_REQUEST.insert(), profile_id=47016, email=args['email'], created_at=time.time()) connection.execute(HDF5_REQUEST_GALAXY.insert(), hdf5_request_id=result.inserted_primary_key, galaxy_id=args['galaxy_id']) for pixel_type in pixel_types: connection.execute(HDF5_REQUEST_PIXEL_TYPE.insert(), hdf5_request_id=result.inserted_primary_key, hdf5_pixel_type_id=pixel_type + 1) # database entries start at 1 for layer in layers:
connection = engine.connect() features, layers, pixel_types = get_features_and_layers_pixeltypes_cmd_line(args) if len(features) == 0 or len(layers) == 0: parser.print_help() exit(1) # Create a new request # Need to make: # HDF5_request # HDF5_request_feature # HDF5_request_galaxy # HDF5_request_layer # HDF5_request_pixel transaction = connection.begin() result = connection.execute(HDF5_REQUEST.insert(), profile_id=47016, email=args['email'], created_at=time.time()) connection.execute(HDF5_REQUEST_GALAXY.insert(), hdf5_request_id=result.inserted_primary_key, galaxy_id=args['galaxy_id']) for pixel_type in pixel_types: connection.execute(HDF5_REQUEST_PIXEL_TYPE.insert(), hdf5_request_id=result.inserted_primary_key, hdf5_pixel_type_id=pixel_type+1) # database entries start at 1 for layer in layers: connection.execute(HDF5_REQUEST_LAYER.insert(), hdf5_request_id=result.inserted_primary_key, hdf5_layer_id=LAYERS[layer]+1) for feature in features: connection.execute(HDF5_REQUEST_FEATURE.insert(), hdf5_request_id=result.inserted_primary_key, hdf5_feature_id=FEATURES[feature]+1) transaction.commit() LOG.info('All done') connection.close()