def run(self): self.parse_args(sys.argv[1:]) self.config = configxml.default_config().config # retrieve app where name = app.name database.connect() database.Apps.find1(name=self.appname) conn = db_base.get_dbconnection() user_count = 0 credit_count = 0 transaction = self._connection.begin() query_cursor = conn.cursor() delete_cursor = conn.cursor() query_cursor.execute('select userid, workunitid from credited_job ') results = query_cursor.fetchall() for result in results: user_count += 1 user_id = result['userid'] area_id = result['workunitid'] area_user = self._connection.execute( select([AREA_USER]).where( and_(AREA_USER.c.userid == user_id, AREA_USER.c.area_id == area_id))).first() if area_user is None: area = self._connection.execute( select([AREA]).where(AREA.c.area_id == area_id)).first() galaxy_id = area['galaxy_id'] if area is None: print 'Area', area_id, 'not found, User', user_id, 'not Credited' else: AREA_USER.insert().values(userid=user_id, area_id=area_id) GALAXY_USER.insert().prefix_with('IGNORE').values( userid=user_id, galaxy_id=galaxy_id) print 'User', user_id, 'Credited for Area', area_id credit_count += 1 delete_cursor.execute("delete from credited_job where userid = " + str(user_id) + " and workunitid = " + str(area_id)) transaction.commit() conn.commit() database.close() self._connection.close() print user_count, 'Users', credit_count, 'Credited'
def run(self): self.parse_args(sys.argv[1:]) self.config = configxml.default_config().config # retrieve app where name = app.name database.connect() database.Apps.find1(name=self.appname) conn = db_base.get_dbconnection() user_count = 0 credit_count = 0 transaction = self._connection.begin() query_cursor = conn.cursor() delete_cursor = conn.cursor() query_cursor.execute('select userid, workunitid from credited_job ') results = query_cursor.fetchall() for result in results: user_count += 1 user_id = result['userid'] area_id = result['workunitid'] area_user = self._connection.execute(select([AREA_USER]).where(and_(AREA_USER.c.userid == user_id, AREA_USER.c.area_id == area_id))).first() if area_user is None: area = self._connection.execute(select([AREA]).where(AREA.c.area_id == area_id)).first() galaxy_id = area['galaxy_id'] if area is None: print 'Area', area_id, 'not found, User', user_id, 'not Credited' else: AREA_USER.insert().values(userid=user_id, area_id=area_id) GALAXY_USER.insert().prefix_with('IGNORE').values(userid=user_id, galaxy_id=galaxy_id) print 'User', user_id, 'Credited for Area', area_id credit_count += 1 delete_cursor.execute("delete from credited_job where userid = " + str(user_id) + " and workunitid = " + str(area_id)) transaction.commit() conn.commit() database.close() self._connection.close() print user_count, 'Users', credit_count, 'Credited'
def check_area_user(galaxy_id, area_group): """ Check the area user """ data = area_group['area_user'] count = 0 for area_user in connection.execute(select([AREA_USER], from_obj=AREA_USER.join(AREA)).where(AREA.c.galaxy_id == galaxy_id).order_by(AREA_USER.c.areauser_id)): compare(data[count][0], area_user[AREA_USER.c.area_id] , 'Area User', count, 'area_id') compare(data[count][1], area_user[AREA_USER.c.userid] , 'Area User', count, 'userid') compare(data[count][2], str(area_user[AREA_USER.c.create_time]), 'Area User', count, 'create_time') count += 1
def store_area_user(connection, galaxy_id, group): """ Store the areas associated with a galaxy """ LOG.info('Storing the area_users') count = connection.execute( select([func.count(AREA_USER.c.areauser_id)], from_obj=AREA_USER.join(AREA)).where( AREA.c.galaxy_id == galaxy_id)).first()[0] data = numpy.zeros(count, dtype=data_type_area_user) count = 0 for area_user in connection.execute( select([AREA_USER], from_obj=AREA_USER.join(AREA)).where( AREA.c.galaxy_id == galaxy_id).order_by( AREA_USER.c.areauser_id)): data[count] = ( area_user[AREA_USER.c.area_id], area_user[AREA_USER.c.userid], str(area_user[AREA_USER.c.create_time]), ) count += 1 group.create_dataset('area_user', data=data, compression='gzip')
def delete_galaxy(connection, galaxy_id): if DRY_RUN: LOG.info('DRY_RUN: deleting galaxy_id: {0}'.format(galaxy_id)) else: transaction = connection.begin() for area_id1 in connection.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id).order_by(AREA.c.area_id)): connection.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0])) connection.execute(AREA_USER.delete().where(AREA_USER.c.area_id == area_id1[0])) connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id)) connection.execute(FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id)) connection.execute(IMAGE_FILTERS_USED.delete().where(IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id)) connection.execute(GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id)) LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id) transaction.commit()
def store_area_user(connection, galaxy_id, group): """ Store the areas associated with a galaxy """ LOG.info('Storing the area_users') count = connection.execute(select([func.count(AREA_USER.c.areauser_id)], from_obj=AREA_USER.join(AREA)).where(AREA.c.galaxy_id == galaxy_id)).first()[0] data = numpy.zeros(count, dtype=data_type_area_user) count = 0 for area_user in connection.execute(select([AREA_USER], from_obj=AREA_USER.join(AREA)).where(AREA.c.galaxy_id == galaxy_id).order_by(AREA_USER.c.areauser_id)): data[count] = ( area_user[AREA_USER.c.area_id], area_user[AREA_USER.c.userid], str(area_user[AREA_USER.c.create_time]), ) count += 1 group.create_dataset('area_user', data=data, compression='gzip')
def check_area_user(galaxy_id, area_group): """ Check the area user """ data = area_group['area_user'] count = 0 for area_user in connection.execute( select([AREA_USER], from_obj=AREA_USER.join(AREA)).where( AREA.c.galaxy_id == galaxy_id).order_by( AREA_USER.c.areauser_id)): compare(data[count][0], area_user[AREA_USER.c.area_id], 'Area User', count, 'area_id') compare(data[count][1], area_user[AREA_USER.c.userid], 'Area User', count, 'userid') compare(data[count][2], str(area_user[AREA_USER.c.create_time]), 'Area User', count, 'create_time') count += 1
def delete_galaxy(connection, galaxy_id): if DRY_RUN: LOG.info('DRY_RUN: deleting galaxy_id: {0}'.format(galaxy_id)) else: transaction = connection.begin() for area_id1 in connection.execute( select([AREA.c.area_id ]).where(AREA.c.galaxy_id == galaxy_id).order_by( AREA.c.area_id)): connection.execute(PIXEL_RESULT.delete().where( PIXEL_RESULT.c.area_id == area_id1[0])) connection.execute( AREA_USER.delete().where(AREA_USER.c.area_id == area_id1[0])) connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id)) connection.execute( FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id)) connection.execute(IMAGE_FILTERS_USED.delete().where( IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id)) connection.execute( GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id)) LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id) transaction.commit()
for file in glob(path_name): if os.path.isfile(file): LOG.info("Processing {0}".format(file)) # Check file size status_info = os.stat(file) if status_info.st_size > 0: # Open the file h5_file = h5py.File(file, 'r') galaxy_group = h5_file['galaxy'] galaxy_id = galaxy_group.attrs['galaxy_id'] # Do we need to do this file count = connection.execute( select([func.count(AREA_USER.c.areauser_id)], from_obj=AREA_USER.join(AREA)).where( AREA.c.galaxy_id == galaxy_id)).first()[0] if count > 0: LOG.info("{0}({1}) has {2} area_user records".format( file, galaxy_id, count)) else: LOG.info("Adding {0}({1})".format(file, galaxy_id, count)) area_group = galaxy_group['area'] area_user = area_group['area_user'] transaction = connection.begin() insert = AREA_USER.insert() for item in area_user: # area_id - 0 # user_id - 1
def assimilate_handler(self, wu, results, canonical_result): """ Process the Results. """ self.logDebug("Start of assimilate_handler for wu %d\n", wu.id) connection = None transaction = None try: if wu.canonical_result: outFile = self.get_file_path(canonical_result) self.area = None if outFile: if os.path.isfile(outFile): pass else: self.logDebug("File [%s] not found\n", outFile) outFile = None if outFile: self.logDebug("Reading File [%s]\n", outFile) start = time.time() connection = ENGINE.connect() transaction = connection.begin() resultCount = self._process_result(connection, outFile, wu) if self.noinsert: transaction.rollback() else: if not resultCount: self.logCritical("No results were found in the output file\n") if self._area_id is None: self.logDebug("The Area was not found\n") else: connection.execute(AREA.update(). where(AREA.c.area_id == self._area_id). values(workunit_id = wu.id, update_time = datetime.datetime.now())) user_id_set = set() for result in results: if result.user and result.validate_state == boinc_db.VALIDATE_STATE_VALID: user_id = result.user.id if user_id not in user_id_set: user_id_set.add(user_id) connection.execute(AREA_USER.delete().where(AREA_USER.c.area_id == self._area_id)) insert = AREA_USER.insert() for user_id in user_id_set: connection.execute(insert, area_id=self._area_id, userid=user_id) time_taken = '{0:.2f}'.format(time.time() - start) self.logDebug("Saving %d results for workunit %d in %s seconds\n", resultCount, wu.id, time_taken) transaction.commit() connection.close() else: self.logCritical("The output file was not found\n") else: self.logDebug("No canonical_result for workunit\n") self.report_errors(wu) except: if transaction is not None: transaction.rollback() if connection is not None: connection.close() print "Unexpected error:", sys.exc_info()[0] traceback.print_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]) self.logCritical("Unexpected error occurred, retrying...\n") return -1 return 0
def assimilate_handler(self, wu, results, canonical_result): """ Process the Results. """ self.logDebug("Start of assimilate_handler for wu %d\n", wu.id) connection = None transaction = None try: if wu.canonical_result: outFile = self.get_file_path(canonical_result) self.area = None if outFile: if os.path.isfile(outFile): pass else: self.logDebug("File [%s] not found\n", outFile) outFile = None if outFile: self.logDebug("Reading File [%s]\n", outFile) start = time.time() connection = ENGINE.connect() transaction = connection.begin() resultCount = self._process_result(connection, outFile, wu) if self.noinsert: transaction.rollback() else: if not resultCount: self.logCritical( "No results were found in the output file\n") if self._area_id is None: self.logDebug("The Area was not found\n") else: connection.execute(AREA.update().where( AREA.c.area_id == self._area_id).values( workunit_id=wu.id, update_time=datetime.datetime.now())) user_id_set = set() for result in results: if result.user and result.validate_state == boinc_db.VALIDATE_STATE_VALID: user_id = result.user.id if user_id not in user_id_set: user_id_set.add(user_id) connection.execute(AREA_USER.delete().where( AREA_USER.c.area_id == self._area_id)) insert = AREA_USER.insert() for user_id in user_id_set: connection.execute(insert, area_id=self._area_id, userid=user_id) time_taken = '{0:.2f}'.format(time.time() - start) self.logDebug( "Saving %d results for workunit %d in %s seconds\n", resultCount, wu.id, time_taken) transaction.commit() connection.close() else: self.logCritical("The output file was not found\n") else: self.logDebug("No canonical_result for workunit\n") self.report_errors(wu) except: if transaction is not None: transaction.rollback() if connection is not None: connection.close() print "Unexpected error:", sys.exc_info()[0] traceback.print_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]) self.logCritical("Unexpected error occurred, retrying...\n") return -1 return 0
# List the files path_name = os.path.join(DIR, "*.hdf5") for file in glob(path_name): if os.path.isfile(file): LOG.info("Processing {0}".format(file)) # Check file size status_info = os.stat(file) if status_info.st_size > 0: # Open the file h5_file = h5py.File(file, 'r') galaxy_group = h5_file['galaxy'] galaxy_id = galaxy_group.attrs['galaxy_id'] # Do we need to do this file count = connection.execute(select([func.count(AREA_USER.c.areauser_id)], from_obj=AREA_USER.join(AREA)).where(AREA.c.galaxy_id == galaxy_id)).first()[0] if count > 0: LOG.info("{0}({1}) has {2} area_user records".format(file, galaxy_id, count)) else: LOG.info("Adding {0}({1})".format(file, galaxy_id, count)) area_group = galaxy_group['area'] area_user = area_group['area_user'] transaction = connection.begin() insert = AREA_USER.insert() for item in area_user: # area_id - 0 # user_id - 1 # create_time - 2 connection.execute(insert, area_id=item[0], userid=item[1], create_time=item[2])
def assimilate_handler(self, wu, results, canonical_result): """ Process the Results. """ self.logDebug("Start of assimilate_handler for wu %d\n", wu.id) connection = None try: if wu.canonical_result: out_file = self.get_file_path(canonical_result) self.area = None if out_file: if os.path.isfile(out_file): pass else: self.logDebug("File [%s] not found\n", out_file) out_file = None if out_file: self.logDebug("Reading File [%s]\n", out_file) start = time.time() connection = ENGINE.connect() result_count = self._process_result(connection, out_file, wu) if self.noinsert: pass else: if not result_count: self.logCritical("No results were found in the output file\n") if self._area_id is None: self.logDebug("The Area was not found\n") else: self._database_queue.append(AREA.update() .where(AREA.c.area_id == self._area_id) .values(workunit_id=wu.id, update_time=datetime.datetime.now())) user_id_set = set() for result in results: if result.user and result.validate_state == boinc_db.VALIDATE_STATE_VALID: user_id = result.user.id if user_id not in user_id_set: user_id_set.add(user_id) self._database_queue.append(AREA_USER.delete().where(AREA_USER.c.area_id == self._area_id)) insert_area_user = AREA_USER.insert() insert_galaxy_user = GALAXY_USER.insert().prefix_with('IGNORE') for user_id in user_id_set: self._database_queue.append(insert_area_user.values(area_id=self._area_id, userid=user_id)) self._database_queue.append(insert_galaxy_user.values(galaxy_id=self._galaxy_id, userid=user_id)) # Copy the file to S3 s3helper = S3Helper() s3helper.add_file_to_bucket(get_sed_files_bucket(), get_key_sed(self._galaxy_name, self._run_id, self._galaxy_id, self._area_id), out_file, reduced_redundancy=True) time_taken = '{0:.2f}'.format(time.time() - start) self.logDebug("Saving %d results for workunit %d in %s seconds\n", result_count, wu.id, time_taken) self._run_pending_db_tasks(connection) connection.close() else: self.logCritical("The output file was not found\n") else: self.logDebug("No canonical_result for workunit\n") self.report_errors(wu) except: if connection is not None: connection.close() print "Unexpected error:", sys.exc_info()[0] traceback.print_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]) self.logCritical("Unexpected error occurred, retrying...\n") return -1 return 0