def run(self):
self.parse_args(sys.argv[1:])
self.config = configxml.default_config().config
# retrieve app where name = app.name
database.connect()
database.Apps.find1(name=self.appname)
conn = db_base.get_dbconnection()
user_count = 0
credit_count = 0
transaction = self._connection.begin()
query_cursor = conn.cursor()
delete_cursor = conn.cursor()
query_cursor.execute('select userid, workunitid from credited_job ')
results = query_cursor.fetchall()
for result in results:
user_count += 1
user_id = result['userid']
area_id = result['workunitid']
area_user = self._connection.execute(
select([AREA_USER]).where(
and_(AREA_USER.c.userid == user_id,
AREA_USER.c.area_id == area_id))).first()
if area_user is None:
area = self._connection.execute(
select([AREA]).where(AREA.c.area_id == area_id)).first()
galaxy_id = area['galaxy_id']
if area is None:
print 'Area', area_id, 'not found, User', user_id, 'not Credited'
else:
AREA_USER.insert().values(userid=user_id, area_id=area_id)
GALAXY_USER.insert().prefix_with('IGNORE').values(
userid=user_id, galaxy_id=galaxy_id)
print 'User', user_id, 'Credited for Area', area_id
credit_count += 1
delete_cursor.execute("delete from credited_job where userid = " +
str(user_id) + " and workunitid = " +
str(area_id))
transaction.commit()
conn.commit()
database.close()
self._connection.close()
print user_count, 'Users', credit_count, 'Credited'
def run(self):
self.parse_args(sys.argv[1:])
self.config = configxml.default_config().config
# retrieve app where name = app.name
database.connect()
database.Apps.find1(name=self.appname)
conn = db_base.get_dbconnection()
user_count = 0
credit_count = 0
transaction = self._connection.begin()
query_cursor = conn.cursor()
delete_cursor = conn.cursor()
query_cursor.execute('select userid, workunitid from credited_job ')
results = query_cursor.fetchall()
for result in results:
user_count += 1
user_id = result['userid']
area_id = result['workunitid']
area_user = self._connection.execute(select([AREA_USER]).where(and_(AREA_USER.c.userid == user_id, AREA_USER.c.area_id == area_id))).first()
if area_user is None:
area = self._connection.execute(select([AREA]).where(AREA.c.area_id == area_id)).first()
galaxy_id = area['galaxy_id']
if area is None:
print 'Area', area_id, 'not found, User', user_id, 'not Credited'
else:
AREA_USER.insert().values(userid=user_id, area_id=area_id)
GALAXY_USER.insert().prefix_with('IGNORE').values(userid=user_id, galaxy_id=galaxy_id)
print 'User', user_id, 'Credited for Area', area_id
credit_count += 1
delete_cursor.execute("delete from credited_job where userid = " + str(user_id) + " and workunitid = " + str(area_id))
transaction.commit()
conn.commit()
database.close()
self._connection.close()
print user_count, 'Users', credit_count, 'Credited'
def check_area_user(galaxy_id, area_group):
"""
Check the area user
"""
data = area_group['area_user']
count = 0
for area_user in connection.execute(select([AREA_USER], from_obj=AREA_USER.join(AREA)).where(AREA.c.galaxy_id == galaxy_id).order_by(AREA_USER.c.areauser_id)):
compare(data[count][0], area_user[AREA_USER.c.area_id] , 'Area User', count, 'area_id')
compare(data[count][1], area_user[AREA_USER.c.userid] , 'Area User', count, 'userid')
compare(data[count][2], str(area_user[AREA_USER.c.create_time]), 'Area User', count, 'create_time')
count += 1
def store_area_user(connection, galaxy_id, group):
"""
Store the areas associated with a galaxy
"""
LOG.info('Storing the area_users')
count = connection.execute(
select([func.count(AREA_USER.c.areauser_id)],
from_obj=AREA_USER.join(AREA)).where(
AREA.c.galaxy_id == galaxy_id)).first()[0]
data = numpy.zeros(count, dtype=data_type_area_user)
count = 0
for area_user in connection.execute(
select([AREA_USER], from_obj=AREA_USER.join(AREA)).where(
AREA.c.galaxy_id == galaxy_id).order_by(
AREA_USER.c.areauser_id)):
data[count] = (
area_user[AREA_USER.c.area_id],
area_user[AREA_USER.c.userid],
str(area_user[AREA_USER.c.create_time]),
)
count += 1
group.create_dataset('area_user', data=data, compression='gzip')
def delete_galaxy(connection, galaxy_id):
if DRY_RUN:
LOG.info('DRY_RUN: deleting galaxy_id: {0}'.format(galaxy_id))
else:
transaction = connection.begin()
for area_id1 in connection.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id).order_by(AREA.c.area_id)):
connection.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
connection.execute(AREA_USER.delete().where(AREA_USER.c.area_id == area_id1[0]))
connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
connection.execute(FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
connection.execute(IMAGE_FILTERS_USED.delete().where(IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
connection.execute(GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))
LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id)
transaction.commit()
def store_area_user(connection, galaxy_id, group):
"""
Store the areas associated with a galaxy
"""
LOG.info('Storing the area_users')
count = connection.execute(select([func.count(AREA_USER.c.areauser_id)], from_obj=AREA_USER.join(AREA)).where(AREA.c.galaxy_id == galaxy_id)).first()[0]
data = numpy.zeros(count, dtype=data_type_area_user)
count = 0
for area_user in connection.execute(select([AREA_USER], from_obj=AREA_USER.join(AREA)).where(AREA.c.galaxy_id == galaxy_id).order_by(AREA_USER.c.areauser_id)):
data[count] = (
area_user[AREA_USER.c.area_id],
area_user[AREA_USER.c.userid],
str(area_user[AREA_USER.c.create_time]),
)
count += 1
group.create_dataset('area_user', data=data, compression='gzip')
def check_area_user(galaxy_id, area_group):
"""
Check the area user
"""
data = area_group['area_user']
count = 0
for area_user in connection.execute(
select([AREA_USER], from_obj=AREA_USER.join(AREA)).where(
AREA.c.galaxy_id == galaxy_id).order_by(
AREA_USER.c.areauser_id)):
compare(data[count][0], area_user[AREA_USER.c.area_id], 'Area User',
count, 'area_id')
compare(data[count][1], area_user[AREA_USER.c.userid], 'Area User',
count, 'userid')
compare(data[count][2], str(area_user[AREA_USER.c.create_time]),
'Area User', count, 'create_time')
count += 1
def delete_galaxy(connection, galaxy_id):
if DRY_RUN:
LOG.info('DRY_RUN: deleting galaxy_id: {0}'.format(galaxy_id))
else:
transaction = connection.begin()
for area_id1 in connection.execute(
select([AREA.c.area_id
]).where(AREA.c.galaxy_id == galaxy_id).order_by(
AREA.c.area_id)):
connection.execute(PIXEL_RESULT.delete().where(
PIXEL_RESULT.c.area_id == area_id1[0]))
connection.execute(
AREA_USER.delete().where(AREA_USER.c.area_id == area_id1[0]))
connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
connection.execute(
FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
connection.execute(IMAGE_FILTERS_USED.delete().where(
IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
connection.execute(
GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))
LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id)
transaction.commit()
for file in glob(path_name):
if os.path.isfile(file):
LOG.info("Processing {0}".format(file))
# Check file size
status_info = os.stat(file)
if status_info.st_size > 0:
# Open the file
h5_file = h5py.File(file, 'r')
galaxy_group = h5_file['galaxy']
galaxy_id = galaxy_group.attrs['galaxy_id']
# Do we need to do this file
count = connection.execute(
select([func.count(AREA_USER.c.areauser_id)],
from_obj=AREA_USER.join(AREA)).where(
AREA.c.galaxy_id == galaxy_id)).first()[0]
if count > 0:
LOG.info("{0}({1}) has {2} area_user records".format(
file, galaxy_id, count))
else:
LOG.info("Adding {0}({1})".format(file, galaxy_id, count))
area_group = galaxy_group['area']
area_user = area_group['area_user']
transaction = connection.begin()
insert = AREA_USER.insert()
for item in area_user:
# area_id - 0
# user_id - 1
def assimilate_handler(self, wu, results, canonical_result):
"""
Process the Results.
"""
self.logDebug("Start of assimilate_handler for wu %d\n", wu.id)
connection = None
transaction = None
try:
if wu.canonical_result:
outFile = self.get_file_path(canonical_result)
self.area = None
if outFile:
if os.path.isfile(outFile):
pass
else:
self.logDebug("File [%s] not found\n", outFile)
outFile = None
if outFile:
self.logDebug("Reading File [%s]\n", outFile)
start = time.time()
connection = ENGINE.connect()
transaction = connection.begin()
resultCount = self._process_result(connection, outFile, wu)
if self.noinsert:
transaction.rollback()
else:
if not resultCount:
self.logCritical("No results were found in the output file\n")
if self._area_id is None:
self.logDebug("The Area was not found\n")
else:
connection.execute(AREA.update().
where(AREA.c.area_id == self._area_id).
values(workunit_id = wu.id, update_time = datetime.datetime.now()))
user_id_set = set()
for result in results:
if result.user and result.validate_state == boinc_db.VALIDATE_STATE_VALID:
user_id = result.user.id
if user_id not in user_id_set:
user_id_set.add(user_id)
connection.execute(AREA_USER.delete().where(AREA_USER.c.area_id == self._area_id))
insert = AREA_USER.insert()
for user_id in user_id_set:
connection.execute(insert, area_id=self._area_id, userid=user_id)
time_taken = '{0:.2f}'.format(time.time() - start)
self.logDebug("Saving %d results for workunit %d in %s seconds\n", resultCount, wu.id, time_taken)
transaction.commit()
connection.close()
else:
self.logCritical("The output file was not found\n")
else:
self.logDebug("No canonical_result for workunit\n")
self.report_errors(wu)
except:
if transaction is not None:
transaction.rollback()
if connection is not None:
connection.close()
print "Unexpected error:", sys.exc_info()[0]
traceback.print_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
self.logCritical("Unexpected error occurred, retrying...\n")
return -1
return 0
def assimilate_handler(self, wu, results, canonical_result):
"""
Process the Results.
"""
self.logDebug("Start of assimilate_handler for wu %d\n", wu.id)
connection = None
transaction = None
try:
if wu.canonical_result:
outFile = self.get_file_path(canonical_result)
self.area = None
if outFile:
if os.path.isfile(outFile):
pass
else:
self.logDebug("File [%s] not found\n", outFile)
outFile = None
if outFile:
self.logDebug("Reading File [%s]\n", outFile)
start = time.time()
connection = ENGINE.connect()
transaction = connection.begin()
resultCount = self._process_result(connection, outFile, wu)
if self.noinsert:
transaction.rollback()
else:
if not resultCount:
self.logCritical(
"No results were found in the output file\n")
if self._area_id is None:
self.logDebug("The Area was not found\n")
else:
connection.execute(AREA.update().where(
AREA.c.area_id == self._area_id).values(
workunit_id=wu.id,
update_time=datetime.datetime.now()))
user_id_set = set()
for result in results:
if result.user and result.validate_state == boinc_db.VALIDATE_STATE_VALID:
user_id = result.user.id
if user_id not in user_id_set:
user_id_set.add(user_id)
connection.execute(AREA_USER.delete().where(
AREA_USER.c.area_id == self._area_id))
insert = AREA_USER.insert()
for user_id in user_id_set:
connection.execute(insert,
area_id=self._area_id,
userid=user_id)
time_taken = '{0:.2f}'.format(time.time() - start)
self.logDebug(
"Saving %d results for workunit %d in %s seconds\n",
resultCount, wu.id, time_taken)
transaction.commit()
connection.close()
else:
self.logCritical("The output file was not found\n")
else:
self.logDebug("No canonical_result for workunit\n")
self.report_errors(wu)
except:
if transaction is not None:
transaction.rollback()
if connection is not None:
connection.close()
print "Unexpected error:", sys.exc_info()[0]
traceback.print_exception(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
self.logCritical("Unexpected error occurred, retrying...\n")
return -1
return 0
# List the files
path_name = os.path.join(DIR, "*.hdf5")
for file in glob(path_name):
if os.path.isfile(file):
LOG.info("Processing {0}".format(file))
# Check file size
status_info = os.stat(file)
if status_info.st_size > 0:
# Open the file
h5_file = h5py.File(file, 'r')
galaxy_group = h5_file['galaxy']
galaxy_id = galaxy_group.attrs['galaxy_id']
# Do we need to do this file
count = connection.execute(select([func.count(AREA_USER.c.areauser_id)], from_obj=AREA_USER.join(AREA)).where(AREA.c.galaxy_id == galaxy_id)).first()[0]
if count > 0:
LOG.info("{0}({1}) has {2} area_user records".format(file, galaxy_id, count))
else:
LOG.info("Adding {0}({1})".format(file, galaxy_id, count))
area_group = galaxy_group['area']
area_user = area_group['area_user']
transaction = connection.begin()
insert = AREA_USER.insert()
for item in area_user:
# area_id - 0
# user_id - 1
# create_time - 2
connection.execute(insert, area_id=item[0], userid=item[1], create_time=item[2])
def assimilate_handler(self, wu, results, canonical_result):
"""
Process the Results.
"""
self.logDebug("Start of assimilate_handler for wu %d\n", wu.id)
connection = None
try:
if wu.canonical_result:
out_file = self.get_file_path(canonical_result)
self.area = None
if out_file:
if os.path.isfile(out_file):
pass
else:
self.logDebug("File [%s] not found\n", out_file)
out_file = None
if out_file:
self.logDebug("Reading File [%s]\n", out_file)
start = time.time()
connection = ENGINE.connect()
result_count = self._process_result(connection, out_file, wu)
if self.noinsert:
pass
else:
if not result_count:
self.logCritical("No results were found in the output file\n")
if self._area_id is None:
self.logDebug("The Area was not found\n")
else:
self._database_queue.append(AREA.update()
.where(AREA.c.area_id == self._area_id)
.values(workunit_id=wu.id, update_time=datetime.datetime.now()))
user_id_set = set()
for result in results:
if result.user and result.validate_state == boinc_db.VALIDATE_STATE_VALID:
user_id = result.user.id
if user_id not in user_id_set:
user_id_set.add(user_id)
self._database_queue.append(AREA_USER.delete().where(AREA_USER.c.area_id == self._area_id))
insert_area_user = AREA_USER.insert()
insert_galaxy_user = GALAXY_USER.insert().prefix_with('IGNORE')
for user_id in user_id_set:
self._database_queue.append(insert_area_user.values(area_id=self._area_id, userid=user_id))
self._database_queue.append(insert_galaxy_user.values(galaxy_id=self._galaxy_id, userid=user_id))
# Copy the file to S3
s3helper = S3Helper()
s3helper.add_file_to_bucket(get_sed_files_bucket(),
get_key_sed(self._galaxy_name, self._run_id, self._galaxy_id, self._area_id),
out_file,
reduced_redundancy=True)
time_taken = '{0:.2f}'.format(time.time() - start)
self.logDebug("Saving %d results for workunit %d in %s seconds\n", result_count, wu.id, time_taken)
self._run_pending_db_tasks(connection)
connection.close()
else:
self.logCritical("The output file was not found\n")
else:
self.logDebug("No canonical_result for workunit\n")
self.report_errors(wu)
except:
if connection is not None:
connection.close()
print "Unexpected error:", sys.exc_info()[0]
traceback.print_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
self.logCritical("Unexpected error occurred, retrying...\n")
return -1
return 0
