Exemple #1
0
    def mark_image(self, in_image_file_name, out_image_file_name, galaxy_id,
                   userid):
        """
        Read the image for the galaxy and generate an image that highlights the areas
        that the specified user has generated results.
        :param in_image_file_name:
        :param out_image_file_name:
        :param galaxy_id:
        :param userid:
        """
        image = Image.open(in_image_file_name, "r").convert("RGBA")
        width, height = image.size
        LOG.info('Width: {0}, Height: {1}'.format(width, height))

        areas = self._connection.execute(
            select([AREA], from_obj=AREA.join(AREA_USER)).where(
                and_(AREA_USER.c.userid == userid,
                     AREA.c.galaxy_id == galaxy_id)).order_by(
                         AREA.c.top_x, AREA.c.top_y))

        for area in areas:
            # LOG.info('top_x: {0}, bottom_x: {1}, top_y: {2}, bottom_y: {3}'.format(area[AREA.c.top_x], area[AREA.c.bottom_x], area[AREA.c.top_y], area[AREA.c.bottom_y]))
            for x in range(area[AREA.c.top_x], area[AREA.c.bottom_x]):
                for y in range(area[AREA.c.top_y], area[AREA.c.bottom_y]):
                    if x < width and y < height:
                        self._mark_pixel(image, x, height - y - 1)

        image.save(out_image_file_name)
Exemple #2
0
def old(iterations):
    p_start = time.time()
    connection = ENGINE.connect()
    db_time = []
    i = 0
    while i < 300:
        start = time.time()
        transaction = connection.begin()

        a = 0
        while a < iterations:
            area = random.randrange(5, 60, 1)
            wu_id = random.randrange(5, 60, 1)
            connection.execute(
                AREA.update().where(AREA.c.area_id == area).values(
                    workunit_id=wu_id, update_time=datetime.datetime.now()))
            a += 1

        sleepytime = random.randrange(80, 140, 1)
        time.sleep(sleepytime / 100.0)

        transaction.commit()
        print 'Time in DB {0}'.format(time.time() - start)
        db_time.append(time.time() - start)
        i += 1

    total = 0
    for dbtime in db_time:
        total += dbtime

    ave = total / len(db_time)
    print 'Total time: {0}'.format(total)
    print 'Ave per transaction: {0}'.format(ave)
    print 'Total program run time: {0}'.format(time.time() - p_start)
Exemple #3
0
def old(iterations):
    p_start = time.time()
    connection = ENGINE.connect()
    db_time = []
    i = 0
    while i < 300:
        start = time.time()
        transaction = connection.begin()

        a = 0
        while a < iterations:
            area = random.randrange(5, 60, 1)
            wu_id = random.randrange(5, 60, 1)
            connection.execute(AREA.update()
                               .where(AREA.c.area_id == area)
                               .values(workunit_id=wu_id, update_time=datetime.datetime.now()))
            a += 1

        sleepytime = random.randrange(80, 140, 1)
        time.sleep(sleepytime/100.0)

        transaction.commit()
        print 'Time in DB {0}'.format(time.time() - start)
        db_time.append(time.time() - start)
        i += 1

    total = 0
    for dbtime in db_time:
        total += dbtime

    ave = total/len(db_time)
    print 'Total time: {0}'.format(total)
    print 'Ave per transaction: {0}'.format(ave)
    print 'Total program run time: {0}'.format(time.time() - p_start)
Exemple #4
0
    def mark_image(self, in_image_file_name, out_image_file_name, galaxy_id, userid):
        """
        Read the image for the galaxy and generate an image that highlights the areas
        that the specified user has generated results.
        :param in_image_file_name:
        :param out_image_file_name:
        :param galaxy_id:
        :param userid:
        """
        image = Image.open(in_image_file_name, "r").convert("RGBA")
        width, height = image.size
        LOG.info('Width: {0}, Height: {1}'.format(width, height))

        areas = self._connection.execute(select([AREA], from_obj=AREA.join(AREA_USER))
                                         .where(and_(AREA_USER.c.userid == userid, AREA.c.galaxy_id == galaxy_id))
                                         .order_by(AREA.c.top_x, AREA.c.top_y))

        for area in areas:
            # LOG.info('top_x: {0}, bottom_x: {1}, top_y: {2}, bottom_y: {3}'.format(area[AREA.c.top_x], area[AREA.c.bottom_x], area[AREA.c.top_y], area[AREA.c.bottom_y]))
            for x in range(area[AREA.c.top_x], area[AREA.c.bottom_x]):
                for y in range(area[AREA.c.top_y], area[AREA.c.bottom_y]):
                    if x < width and y < height:
                        self._mark_pixel(image, x, height - y - 1)

        image.save(out_image_file_name)
Exemple #5
0
    def _create_areas(self, pix_y):
        """
        Create a area - we try to make them squares, but they aren't as the images have dead zones
        """
        area_insert = AREA.insert()
        pixel_result_insert = PIXEL_RESULT.insert()
        pix_x = 0
        while pix_x < self._end_x:
            max_x, pixels = self._get_pixels(pix_x, pix_y)
            if len(pixels) > 0:
                area = Area(pix_x, pix_y, max_x, min(pix_y + WG_ROW_HEIGHT, self._end_y))
                result1 = self._connection.execute(area_insert.values(galaxy_id=self._galaxy_id,
                                                                      top_x=area.top_x,
                                                                      top_y=area.top_y,
                                                                      bottom_x=area.bottom_x,
                                                                      bottom_y=area.bottom_y))
                area.area_id = result1.inserted_primary_key[0]

                for pixel in pixels:
                    result2 = self._connection.execute(pixel_result_insert.values(galaxy_id=self._galaxy_id,
                                                                                  area_id=area.area_id,
                                                                                  y=pixel.y,
                                                                                  x=pixel.x))

                    pixel.pixel_id = result2.inserted_primary_key[0]
                    self._pixel_count += 1

                # Write the pixels
                self._create_output_file(area, pixels)
                self._work_units_added += 1

            pix_x = max_x + 1
def remove_database_entries(connection, galaxy_id):
    connection.execute(
        PIXEL_RESULT.delete().where(PIXEL_RESULT.c.galaxy_id == galaxy_id))
    connection.execute(IMAGE_FILTERS_USED.delete().where(
        IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
    connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
    connection.execute(
        FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
    connection.execute(GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))
Exemple #7
0
def lock(time_p):
    connection = ENGINE.connect()
    transaction = connection.begin()
    i = 999
    while True:
        wu_id = random.randrange(5, 60, 1)
        connection.execute(AREA.update().where(AREA.c.area_id == wu_id).values(
            workunit_id=wu_id, update_time=datetime.datetime.now()))
        connection.execute(PIXEL_RESULT.update().where(
            PIXEL_RESULT.c.pxresult_id == wu_id).values(y=i, x=i))
        i += 1
        if i > 100000:
            break
    transaction.rollback()
def delete_galaxy(connection, galaxy_id):
    if DRY_RUN:
        LOG.info('DRY_RUN: deleting galaxy_id: {0}'.format(galaxy_id))
    else:
        transaction = connection.begin()
        for area_id1 in connection.execute(select([AREA.c.area_id]).where(AREA.c.galaxy_id == galaxy_id).order_by(AREA.c.area_id)):
            connection.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.area_id == area_id1[0]))
            connection.execute(AREA_USER.delete().where(AREA_USER.c.area_id == area_id1[0]))

        connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
        connection.execute(FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
        connection.execute(IMAGE_FILTERS_USED.delete().where(IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
        connection.execute(GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))

        LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id)
        transaction.commit()
Exemple #9
0
def lock(time_p):
    connection = ENGINE.connect()
    transaction = connection.begin()
    i = 999
    while True:
        wu_id = random.randrange(5, 60, 1)
        connection.execute(
            AREA.update()
                .where(AREA.c.area_id == wu_id)
                .values(workunit_id=wu_id, update_time=datetime.datetime.now()))
        connection.execute(PIXEL_RESULT.update().where(PIXEL_RESULT.c.pxresult_id == wu_id).values(y=i,
                                                                                                   x=i))
        i += 1
        if i > 100000:
            break
    transaction.rollback()
Exemple #10
0
def delete_galaxy(connection, galaxy_id):
    if DRY_RUN:
        LOG.info('DRY_RUN: deleting galaxy_id: {0}'.format(galaxy_id))
    else:
        transaction = connection.begin()
        for area_id1 in connection.execute(
                select([AREA.c.area_id
                        ]).where(AREA.c.galaxy_id == galaxy_id).order_by(
                            AREA.c.area_id)):
            connection.execute(PIXEL_RESULT.delete().where(
                PIXEL_RESULT.c.area_id == area_id1[0]))
            connection.execute(
                AREA_USER.delete().where(AREA_USER.c.area_id == area_id1[0]))

        connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
        connection.execute(
            FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
        connection.execute(IMAGE_FILTERS_USED.delete().where(
            IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
        connection.execute(
            GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))

        LOG.info('Galaxy with galaxy_id of %d was deleted', galaxy_id)
        transaction.commit()
    def assimilate_handler(self, wu, results, canonical_result):
        """
        Process the Results.
        """
        self.logDebug("Start of assimilate_handler for wu %d\n", wu.id)
        connection = None
        transaction = None
        try:
            if wu.canonical_result:
                outFile = self.get_file_path(canonical_result)
                self.area = None
                if outFile:
                     if os.path.isfile(outFile):
                          pass
                     else:
                         self.logDebug("File [%s] not found\n", outFile)
                         outFile = None

                if outFile:
                    self.logDebug("Reading File [%s]\n", outFile)
                    start = time.time()
                    connection = ENGINE.connect()
                    transaction = connection.begin()
                    resultCount = self._process_result(connection, outFile, wu)
                    if self.noinsert:
                        transaction.rollback()
                    else:
                        if not resultCount:
                            self.logCritical("No results were found in the output file\n")

                        if self._area_id is None:
                            self.logDebug("The Area was not found\n")
                        else:
                            connection.execute(AREA.update().
                                where(AREA.c.area_id == self._area_id).
                                values(workunit_id = wu.id, update_time = datetime.datetime.now()))

                            user_id_set = set()
                            for result in results:
                                if result.user and result.validate_state == boinc_db.VALIDATE_STATE_VALID:
                                    user_id = result.user.id
                                    if user_id not in user_id_set:
                                        user_id_set.add(user_id)

                            connection.execute(AREA_USER.delete().where(AREA_USER.c.area_id == self._area_id))
                            insert = AREA_USER.insert()
                            for user_id in user_id_set:
                                connection.execute(insert, area_id=self._area_id, userid=user_id)

                        time_taken = '{0:.2f}'.format(time.time() - start)
                        self.logDebug("Saving %d results for workunit %d in %s seconds\n", resultCount, wu.id, time_taken)
                        transaction.commit()
                    connection.close()
                else:
                    self.logCritical("The output file was not found\n")
            else:
                self.logDebug("No canonical_result for workunit\n")
                self.report_errors(wu)
        except:
            if transaction is not None:
                transaction.rollback()
            if connection is not None:
                connection.close()
            print "Unexpected error:", sys.exc_info()[0]
            traceback.print_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
            self.logCritical("Unexpected error occurred, retrying...\n")
            return -1

        return 0
Exemple #12
0
    def assimilate_handler(self, wu, results, canonical_result):
        """
        Process the Results.
        """
        self.logDebug("Start of assimilate_handler for wu %d\n", wu.id)
        connection = None
        transaction = None
        try:
            if wu.canonical_result:
                outFile = self.get_file_path(canonical_result)
                self.area = None
                if outFile:
                    if os.path.isfile(outFile):
                        pass
                    else:
                        self.logDebug("File [%s] not found\n", outFile)
                        outFile = None

                if outFile:
                    self.logDebug("Reading File [%s]\n", outFile)
                    start = time.time()
                    connection = ENGINE.connect()
                    transaction = connection.begin()
                    resultCount = self._process_result(connection, outFile, wu)
                    if self.noinsert:
                        transaction.rollback()
                    else:
                        if not resultCount:
                            self.logCritical(
                                "No results were found in the output file\n")

                        if self._area_id is None:
                            self.logDebug("The Area was not found\n")
                        else:
                            connection.execute(AREA.update().where(
                                AREA.c.area_id == self._area_id).values(
                                    workunit_id=wu.id,
                                    update_time=datetime.datetime.now()))

                            user_id_set = set()
                            for result in results:
                                if result.user and result.validate_state == boinc_db.VALIDATE_STATE_VALID:
                                    user_id = result.user.id
                                    if user_id not in user_id_set:
                                        user_id_set.add(user_id)

                            connection.execute(AREA_USER.delete().where(
                                AREA_USER.c.area_id == self._area_id))
                            insert = AREA_USER.insert()
                            for user_id in user_id_set:
                                connection.execute(insert,
                                                   area_id=self._area_id,
                                                   userid=user_id)

                        time_taken = '{0:.2f}'.format(time.time() - start)
                        self.logDebug(
                            "Saving %d results for workunit %d in %s seconds\n",
                            resultCount, wu.id, time_taken)
                        transaction.commit()
                    connection.close()
                else:
                    self.logCritical("The output file was not found\n")
            else:
                self.logDebug("No canonical_result for workunit\n")
                self.report_errors(wu)
        except:
            if transaction is not None:
                transaction.rollback()
            if connection is not None:
                connection.close()
            print "Unexpected error:", sys.exc_info()[0]
            traceback.print_exception(sys.exc_info()[0],
                                      sys.exc_info()[1],
                                      sys.exc_info()[2])
            self.logCritical("Unexpected error occurred, retrying...\n")
            return -1

        return 0
Exemple #13
0
    def assimilate_handler(self, wu, results, canonical_result):
        """
        Process the Results.
        """
        self.logDebug("Start of assimilate_handler for wu %d\n", wu.id)
        connection = None
        try:
            if wu.canonical_result:
                out_file = self.get_file_path(canonical_result)
                self.area = None
                if out_file:
                    if os.path.isfile(out_file):
                        pass
                    else:
                        self.logDebug("File [%s] not found\n", out_file)
                        out_file = None

                if out_file:
                    self.logDebug("Reading File [%s]\n", out_file)
                    start = time.time()
                    connection = ENGINE.connect()

                    result_count = self._process_result(connection, out_file, wu)
                    if self.noinsert:
                        pass
                    else:
                        if not result_count:
                            self.logCritical("No results were found in the output file\n")

                        if self._area_id is None:
                            self.logDebug("The Area was not found\n")
                        else:
                            self._database_queue.append(AREA.update()
                                                            .where(AREA.c.area_id == self._area_id)
                                                            .values(workunit_id=wu.id, update_time=datetime.datetime.now()))

                            user_id_set = set()
                            for result in results:
                                if result.user and result.validate_state == boinc_db.VALIDATE_STATE_VALID:
                                    user_id = result.user.id
                                    if user_id not in user_id_set:
                                        user_id_set.add(user_id)

                            self._database_queue.append(AREA_USER.delete().where(AREA_USER.c.area_id == self._area_id))

                            insert_area_user = AREA_USER.insert()
                            insert_galaxy_user = GALAXY_USER.insert().prefix_with('IGNORE')
                            for user_id in user_id_set:
                                self._database_queue.append(insert_area_user.values(area_id=self._area_id, userid=user_id))
                                self._database_queue.append(insert_galaxy_user.values(galaxy_id=self._galaxy_id, userid=user_id))

                            # Copy the file to S3
                            s3helper = S3Helper()
                            s3helper.add_file_to_bucket(get_sed_files_bucket(),
                                                        get_key_sed(self._galaxy_name, self._run_id, self._galaxy_id, self._area_id),
                                                        out_file,
                                                        reduced_redundancy=True)

                        time_taken = '{0:.2f}'.format(time.time() - start)
                        self.logDebug("Saving %d results for workunit %d in %s seconds\n", result_count, wu.id, time_taken)

                    self._run_pending_db_tasks(connection)
                    connection.close()
                else:
                    self.logCritical("The output file was not found\n")
            else:
                self.logDebug("No canonical_result for workunit\n")
                self.report_errors(wu)
        except:
            if connection is not None:
                connection.close()
            print "Unexpected error:", sys.exc_info()[0]
            traceback.print_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
            self.logCritical("Unexpected error occurred, retrying...\n")
            return -1

        return 0
Exemple #14
0
def remove_database_entries(connection, galaxy_id):
    connection.execute(PIXEL_RESULT.delete().where(PIXEL_RESULT.c.galaxy_id == galaxy_id))
    connection.execute(IMAGE_FILTERS_USED.delete().where(IMAGE_FILTERS_USED.c.galaxy_id == galaxy_id))
    connection.execute(AREA.delete().where(AREA.c.galaxy_id == galaxy_id))
    connection.execute(FITS_HEADER.delete().where(FITS_HEADER.c.galaxy_id == galaxy_id))
    connection.execute(GALAXY.delete().where(GALAXY.c.galaxy_id == galaxy_id))