Exemple #1
0
 def __init__(self, total_p):
     self.cursor = 1
     self.total = total_p
     self.bar = Bar(max_value=total_p)
     self.bar.cursor.clear_lines(10)
     self.bar.cursor.save()
     self.bar.draw(value=self.cursor)
Exemple #2
0
    def _progress_loop(self, request_method, url, body):
        response = request_method(url, data=json.dumps(body))
        body = response.json()

        max_tasks = body["num_total_tasks"]

        bar = Bar(max_value=max_tasks, title="Completed Tasks",
                  num_rep="percentage", filled_color=2)

        n = ProgressTree(term=self.t)
        n.cursor.clear_lines(self.t.height - 1)

        while True:
            response = request_method(url, data=json.dumps(body))
            body = response.json()

            n.cursor.restore()
            n.cursor.clear_lines(self.t.height - 1)
            n.cursor.save()
            bar.draw(value=body["num_finished_tasks"])

            presp = body
            presp["energy_history"] = sorted(presp["energy_history"])[:10]
            del presp["best_location"]
            print(json.dumps(presp, indent=4))

            if response.status_code == 200:
                return response

            sleep(2.0)
Exemple #3
0
def check_mail(step=3, host=None, username=None): # check emails per `step` minutes
    pymail = mail(host=host, username=username)
    info = pymail.mails_info()
    sleep_time = step/100
    #sleep_time = 0.02 # for test
    try:
        while 1:
            bar = Bar(max_value=10, fallback=True, filled_color=1, title='Check Emails...')
            bar.cursor.clear_lines(2)
            bar.cursor.save()
            for i in range(11):
                sleep(sleep_time)
                # We restore the cursor to saved position before writing
                bar.cursor.restore()
                # Now we draw the bar
                bar.draw(value=i)
            if pymail.mails_info()[0] >= info[0]:
                info = pymail.mails_info()
                win = tk.Tk()
                win.title("Mail Checking")
                win.resizable(False, False)
                win.update()
                scrn_width, scrn_height = win.maxsize()
                win.geometry('200x70+%d+%d'%((scrn_width-200)/2,(scrn_height-65)/2))
                warn_gui(win)
                playsound()
                win.mainloop()
                print('\033[0;32mChecked...\033[0m')
                pymail.quit()
                return
    except(KeyboardInterrupt, SystemExit):
        pymail.quit()
Exemple #4
0
def process_request(obj):
    # check if year is mentioned
    if obj.year:
        _year = obj.year
    else:
        _year = datetime.datetime.now().strftime("%Y")

    if obj.month:
        _month = obj.month.title()
    else:
        _month = datetime.datetime.now().strftime("%B")

    if obj.day:
        _day = obj.day
    else:
        _day = datetime.datetime.now().strftime("%d")

    # create a database connection
    path = os.path.join(os.path.dirname(__file__, ), '..', '..', 'data',
                        _year + '.db')
    conn = sqlite3.connect(path)

    # create object of Bar
    prog = Bar(filled_color=2, title=u'Fetching details , please wait ....')

    # create cursor object and clear lines
    prog.cursor.clear_lines(2)

    # save the state of the cursor
    prog.cursor.save()

    for i in range(101):
        time.sleep(0.02)
        prog.cursor.restore()
        prog.draw(i)

    if obj.day:
        #display records
        cursor = conn.execute("SELECT * FROM '%s' WHERE DAY='%d'" %
                              (_month, int(_day)))
    else:
        cursor = conn.execute("SELECT * FROM '%s'" % _month)
    if cursor:
        print "Successully fetched details"
    else:
        print "No details to fetch"
        sys.exit()
    # displaying results in nice tabular format [TODO]
    headers = ['id', 'day', 'amount', 'tag']
    data = []
    for row in cursor:
        data.append(row)

    # displaying the table
    print tabulate(data, headers, tablefmt="fancy_grid")

    # graphical representations [TODO]
    # close the database connection
    conn.close()
Exemple #5
0
 def __init__(self, title='Progress', max_value=100):
     try:
         from progressive.bar import Bar
         self.support_progressive = True
         self.bar = Bar(max_value=max_value, title='Receiving')
         self.bar.cursor.clear_lines(1)
         self.bar.cursor.save()
     except ImportError:
         self.support_progressive = False
         from progress.bar import Bar
         self.bar = Bar('Receiving', max=max_value)
Exemple #6
0
    def _draw(self, tree, indent=0):
        """Recurse through ``tree`` and draw all nodes"""
        if all([isinstance(tree, dict), type(tree) != BarDescriptor]):
            for k, v in sorted(tree.items()):
                bar_desc, subdict = v[0], v[1]

                args = [self.cursor.term] + bar_desc.get("args", [])
                kwargs = dict(title_pos="above", indent=indent, title=k)
                kwargs.update(bar_desc.get("kwargs", {}))

                b = Bar(*args, **kwargs)
                b.draw(value=bar_desc["value"].value, flush=False)

                self._draw(subdict, indent=indent + self.indent)
Exemple #7
0
def process_request(obj):
    # check if year is mentioned
    if obj.year:
        _year = obj.year
    else:
        _year = datetime.datetime.now().strftime("%Y")

    if obj.month:
        _month = obj.month.title()
    else:
        _month = datetime.datetime.now().strftime("%B")

    # create a database connection
    path = os.path.join(os.path.dirname(__file__, ), '..', '..', 'data',
                        _year + '.db')
    conn = sqlite3.connect(path)

    # create object of Bar
    prog = Bar(filled_color=2, title=u'Fetching details , please wait ....')

    # create cursor object and clear lines
    prog.cursor.clear_lines(2)

    # save the state of the cursor
    prog.cursor.save()

    for i in range(101):
        time.sleep(0.02)
        prog.cursor.restore()
        prog.draw(i)

    # fetching month records
    cursor = conn.execute("SELECT * FROM '%s'" % _month)

    if cursor:
        print "Successully fetched details"
    else:
        print "No details to fetch"
        sys.exit()

    _total = 0
    # displaying results in nice tabular format [TODO]
    for row in cursor:
        _total += row[2]

    print "total expenses of month {} = {}".format(_month, str(_total))
    # graphical representations [TODO]
    # close the database connection
    conn.close()
def SearchByKeyword(keyword):
    print '=>Search by keyword: %s' % keyword
    client = MongoClient('localhost', 27017)
    db = client['YFCC100M']
    collection = db['dataset_with_tag']

    query = {'$and': [
        {'User_tags': {'$in': [keyword]}},
        {'Longitude': {'$ne': ''}},
        {'Latitude': {'$ne': ''}}
    ]}

    fields = {}
    fields['User_tags'] = True
    fields['Longitude'] = True
    fields['Latitude'] = True

    cursor = collection.find(query, fields)

    total = cursor.count()
    print "There is %d documents about %s." % (total, keyword)
    print
    print '=>Runing point_in_polygon:'

    bbox_dict = Get_area_list()
    print
    user_tags_list = []
    locations_list = []

    now = 0  # progress
    bar = Bar(max_value=total)
    bar.cursor.clear_lines(2)
    bar.cursor.save()

    for doc in cursor:
        lon = doc['Longitude']
        lat = doc['Latitude']
        user_tags = doc['User_tags']

        country = point_in_polygon_with_shapely(bbox_dict, lon, lat)
        now += 1
        bar.cursor.restore()
        bar.draw(value=now)

        if country != None:
            user_tags_list.append(user_tags)
            locations_list.append(country)

    return user_tags_list, locations_list
Exemple #9
0
class prog_bar(object):
    def __init__(self, total_p):
        self.cursor = 1
        self.total = total_p
        self.bar = Bar(max_value=total_p)
        self.bar.cursor.clear_lines(10)
        self.bar.cursor.save()
        self.bar.draw(value=self.cursor)

    def new_page(self, cnt):
        self.cursor += cnt
        self.bar.cursor.restore()
        self.bar.draw(value=self.cursor)

    def reflash(self, time, size, in_q, wait_q, out_q, urlc):
        self.bar.cursor.restore()
        self.bar.draw(value=self.cursor)
        print "-------------"
        print "done:" + str(size) + "  "
        print "in_q:" + str(in_q) + "  "
        print "wait_q:" + str(wait_q) + "  "
        print "out_q:" + str(out_q) + "  "
        print "spent: " + str(int(time / 60)) + " min" + "  "
        print "rest: " + str(int(time / size * urlc / 60)) + " min"

    def reflash_r(self, f_cnt, q_size, c_cnt, curren_f, filename, e_cnt):
        self.bar.cursor.restore()
        self.bar.draw(value=self.cursor)
        print "-------------"
        print "fetch:" + str(f_cnt) + "  "
        print "q_size:" + str(q_size) + "  "
        print "commit: " + str(c_cnt) + "  "
        print "curren_f: " + str(curren_f) + "  "
        print "file_size: " + str(os.path.getsize(filename)) + "  "
        print "error_count: " + str(e_cnt) + "  "

    def get_stat(self, done, time, filename, wait_q, e_cnt):
        f_size = os.path.getsize(filename)
        self.bar.cursor.restore()
        self.bar.draw(value=self.cursor)
        print "done: " + str(done) + "     file_size: " + str(int(f_size))
        print "total_size: " + str(
            int(f_size / self.cursor * (self.total - self.cursor)))
        print "spent: " + str(int(time / 60)) + " min"
        print "rest: " + str(
            int(time / self.cursor * (self.total - self.cursor) / 60)) + " min"
        print "wait_url: " + str(wait_q)
        print "error_count: " + str(e_cnt)
Exemple #10
0
class prog_bar(object):

    def __init__(self, total_p):
        self.cursor=1
        self.total=total_p
        self.bar = Bar(max_value=total_p)
        self.bar.cursor.clear_lines(10)
        self.bar.cursor.save()
        self.bar.draw(value=self.cursor)

    def new_page(self,cnt):
        self.cursor+=cnt
        self.bar.cursor.restore()
        self.bar.draw(value=self.cursor)

    def reflash(self,time,size,in_q,wait_q,out_q,urlc):
        self.bar.cursor.restore()
        self.bar.draw(value=self.cursor)
        print "-------------"
    	print "done:"+str(size)+"  "
        print "in_q:"+str(in_q)+"  "
        print "wait_q:"+str(wait_q)+"  "
        print "out_q:"+str(out_q)+"  "
    	print "spent: "+str(int(time/60))+" min"+"  "
    	print "rest: "+str(int(time/size*urlc/60))+" min"


    def reflash_r(self,f_cnt,q_size,c_cnt,curren_f,filename,e_cnt):
        self.bar.cursor.restore()
        self.bar.draw(value=self.cursor)
        print "-------------"
    	print "fetch:"+str(f_cnt)+"  "
        print "q_size:"+str(q_size)+"  "
        print "commit: "+str(c_cnt)+"  "
    	print "curren_f: "+str(curren_f)+"  "
    	print "file_size: "+str(os.path.getsize(filename))+"  "
        print "error_count: "+str(e_cnt)+"  "

    def get_stat(self,done,time,filename,wait_q,e_cnt):
        f_size=os.path.getsize(filename)
        self.bar.cursor.restore()
        self.bar.draw(value=self.cursor)
        print "done: "+str(done)+"     file_size: "+str(int(f_size))
        print "total_size: "+str(int(f_size/self.cursor*(self.total-self.cursor)))
    	print "spent: "+str(int(time/60))+" min"
    	print "rest: "+str(int(time/self.cursor*(self.total-self.cursor)/60))+" min"
        print "wait_url: "+str(wait_q)
        print "error_count: "+str(e_cnt)
Exemple #11
0
def start_attack():
    import MySQLdb
    global avalible_pass

    Log.log_info('Starting attack...')
    bar = Bar(max_value=len(user_name_list) * len(pass_list))
    bar.cursor.clear_lines(2)
    bar.cursor.save()
    i = 0

    for user_name in user_name_list:
        for password in pass_list:
            try:
                access = MySQLdb.connect(host=host,
                                         port=port,
                                         user=user_name,
                                         passwd=password)
                database_access = access.cursor()
                # print 'username:'******'Access denied for user' in str(exception):
                    pass
                    # Log.log_error('Access denied')
                else:
                    Log.log_error(str(exception))
                    bar.cursor.clear_lines(2)
                    sys.exit()
            finally:
Exemple #12
0
 def __init__(self, total_p):
     self.cursor=1
     self.total=total_p
     self.bar = Bar(max_value=total_p)
     self.bar.cursor.clear_lines(10)
     self.bar.cursor.save()
     self.bar.draw(value=self.cursor)
Exemple #13
0
    def _draw(self, tree, indent=0):
        """Recurse through ``tree`` and draw all nodes"""
        if all([
            isinstance(tree, dict),
            type(tree) != BarDescriptor
        ]):
            for k, v in sorted(tree.items()):
                bar_desc, subdict = v[0], v[1]

                args = [self.cursor.term] + bar_desc.get("args", [])
                kwargs = dict(title_pos="above", indent=indent, title=k)
                kwargs.update(bar_desc.get("kwargs", {}))

                b = Bar(*args, **kwargs)
                b.draw(value=bar_desc["value"].value, flush=False)

                self._draw(subdict, indent=indent + self.indent)
Exemple #14
0
 def get_bar(title, max_value):
     bar = self._bar if hasattr(self, '_bar') else None
     if bar == None or bar.title != title:
         bar = Bar(max_value=max_value, title=title)
         bar.cursor.clear_lines(2)  # Make some room
         bar.cursor.save()  # Mark starting line
     self._bar = bar
     return bar
Exemple #15
0
 def do_progress(self, args):
     """show progressbar"""
     if self.hydra_running():
         pass
     else:
         print('[x] MiniHydra is not running!')
         return 
     
     try:
         bar = Bar(max_value=self._hydra.get_total_size())
         bar.cursor.clear_lines(2)
         bar.cursor.save()
         while True:
             sleep(0.1)
             bar.cursor.restore()
             bar.draw(value=self._hydra.get_current_pos())
         
     except KeyboardInterrupt:
         pass
Exemple #16
0
def main():
    print('Start time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))
    stockid.deleteContent('yield.txt')
    stockid.gen_stockid()
    f = open('stockid.txt', 'r')
    result = list()

    line_count = id_count('stockid.txt')

    y = open('yield.txt', 'a')
    y.write(" ID   Price  Avg. SD\n")

    count = 0

    bar = Bar(max_value=line_count, fallback=True)
    bar.cursor.clear_lines(2)
    bar.cursor.save()

    for line in f.readlines():
        line = line.strip()
        y = open('yield.txt', 'a')
        # check the yield rate is >= 6.25%
        stock_yield = parse_stock.get_average_dividend(int(line))
        stock_price = parse_stock.get_current_price(int(line))
        historical_price = parse_stock.historical_price(int(line))
        recent_PER = parse_stock.get_recent_PER(int(line))
        total_volume = parse_stock.get_current_volume(int(line))
        time.sleep(0.2)
        if (stock_price == None or historical_price == 0.0):
            #y.write(line + str('  *****error*****   ') + stock_price + str('   historical_price:') + str(historical_price) + str('\n'))
            continue
        if (stock_yield / float(stock_price) >= 0.0625 and  # rule 1
                float(stock_price) / historical_price <= 0.6 and  # rule 2
                float(stock_price) >= 10 and  # rule 3
                0 < float(stock_price) / recent_PER <= 15 and  # rule 4
                int(total_volume) >= 50):  # rule 5
            y.write(line + str('  ') + stock_price + str('   ') +
                    str(stock_yield) + str('\n'))
        count = count + 1
        bar.cursor.restore()
        bar.draw(value=count)
    print('Finish time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))
Exemple #17
0
def simple():
    """Simple example using just the Bar class
    This example is intended to show usage of the Bar class at the lowest
    level.
    """
    MAX_VALUE = 100

    # Create our test progress bar
    bar = Bar(max_value=MAX_VALUE, fallback=True)

    bar.cursor.clear_lines(2)
    # Before beginning to draw our bars, we save the position
    #   of our cursor so we can restore back to this position before writing
    #   the next time.
    bar.cursor.save()
    for i in range(MAX_VALUE + 1):
        sleep(0.1 * random.random())
        # We restore the cursor to saved position before writing
        bar.cursor.restore()
        # Now we draw the bar
        bar.draw(value=i)
Exemple #18
0
def simple():
    """Simple example using just the Bar class
    This example is intended to show usage of the Bar class at the lowest
    level.
    """
    MAX_VALUE = 100

    # Create our test progress bar
    bar = Bar(max_value=MAX_VALUE, fallback=True)

    bar.cursor.clear_lines(2)
     # Before beginning to draw our bars, we save the position
    #   of our cursor so we can restore back to this position before writing
    #   the next time.
    bar.cursor.save()
    for i in range(MAX_VALUE + 1):
        sleep(0.1 * random.random())
        # We restore the cursor to saved position before writing
        bar.cursor.restore()
        # Now we draw the bar
        bar.draw(value=i)
Exemple #19
0
def bar(response, chunk_size=8192, filename="Seicmic data"):
    if response.headers['Content-Length'] == None:
        print("Downloading data to %s......" % filename)
        chunk_all = response.read()
        if type(chunk_all) != str:
            itype = 'bytes'
        else:
            itype = 'str'
#        print("No response of server")
#        sys.exit(1)
    else:
        total_size = response.headers['Content-Length'].strip()
        total_size = int(total_size)
        cycle = int(total_size/chunk_size)+1
        bytes_so_far = 0

        MAX_VALUE = 100

        bar = Bar(max_value=MAX_VALUE, num_rep="percentage", fallback=True)
        
        bar.cursor.clear_lines(2)
        bar.cursor.save()
        for i in range(cycle):
            chunk = response.read(chunk_size)
            if i == 0:
                if type(chunk) != str:
                    chunk_all = b''
                    itype = 'bytes'
                else:
                    chunk_all = ''
                    itype = 'str'
            bytes_so_far += len(chunk)
            chunk_all += chunk
            percentage = int(bytes_so_far*100/total_size)
            bar.cursor.restore()
            bar.draw(value=percentage)
            print('%dbyte/%dbyte' %(bytes_so_far,total_size))
    return chunk_all, itype
Exemple #20
0
class ProgressBar(object):
    def __init__(self, title='Progress', max_value=100):
        try:
            from progressive.bar import Bar
            self.support_progressive = True
            self.bar = Bar(max_value=max_value, title='Receiving')
            self.bar.cursor.clear_lines(1)
            self.bar.cursor.save()
        except ImportError:
            self.support_progressive = False
            from progress.bar import Bar
            self.bar = Bar('Receiving', max=max_value)

    def next(self, count=0):
        if self.support_progressive:
            self.bar.cursor.restore()
            self.bar.draw(value=count)
        else:
            self.bar.next()

    def finish(self):
        if not self.support_progressive:
            self.bar.finish()
Exemple #21
0
#!/usr/bin/env python3
# -*- encoding: utf8 -*-
# Example: python -m scripts.glyph_model_convert FILE1 FILE2
import sys
import json
from importlib import import_module
from progressive.bar import Bar

sys.path.append('src/cv')
GlyphModel = import_module('glyph_model').GlyphModel

with open(sys.argv[1], 'r') as in_fp, open(sys.argv[2], 'w') as out_fp:
    file_js = json.load(in_fp)
    result = {}

    bar = Bar(max_value=len(file_js), fallback=True)
    bar.cursor.clear_lines(2)
    bar.cursor.save()
    i = 0
    for char in file_js:
        glyph = file_js[char]
        try:
            glyph_model = GlyphModel(glyph)
            result[char] = glyph_model.get_model_object()
        except Exception as e:
            print('In file "' + sys.argv[1] + '", gid "' + char + '":')
            print(e)
        i += 1
        bar.cursor.restore()
        bar.draw(value=i)
    json.dump(result, out_fp)
Exemple #22
0
            expcss = 150. * numb  # s
            magab_zeros.append(
                csstpkg.MagAB_Zero(Gain, cssband, expcss, TelArea))

        print('Filter scheme flag:', schemecode)

        namelists = map(lambda mag, magerr, snr, aband: \
                            [mag + aband, magerr+aband, snr+aband], \
                        ['Magsim_'] * len(cssbands), ['ErrMag_'] * len(cssbands), ['SNR_'] * len(cssbands), cssbands)
        colnames = ['ID', 'Z_BEST'] + list(itertools.chain(*namelists))
        OutCat = Table(names=colnames)
        # print(OutCat)
        NColOut = len(OutCat.colnames)

        if IfProgBarOn == True:
            bar = Bar(max_value=len(CatOfTile), empty_color=7, filled_color=4)
            bar.cursor.clear_lines(2)  # Make some room
            bar.cursor.save()  # Mark starting line

        for i in range(len(CatOfTile)):

            # if DebugTF == True:
            # print(CatOfTile[i]['IDENT'])
            a = CatOfTile[i]['a_image_css']
            b = CatOfTile[i]['b_image_css']
            theta = CatOfTile[i]['theta_image']
            ident = str(CatOfTile[i]['IDENT'])

            # Cut a window of the object as objwind
            # cutwidrad = int((a*math.cos(theta/180.*math.pi)+b*abs(math.sin(theta/180.*math.pi)))*5)
            # cutheirad = int((a*abs(math.sin(theta/180.*math.pi))+b*math.cos(theta/180.*math.pi))*5)
Exemple #23
0
import serial
from blessings import Terminal
from progressive.bar import Bar

if len(sys.argv) == 1:
    print("1st arg: serial port e.g. : /dev/tty. device ")
    print("2nd arg: parse string e.g. : \'$A1:\'")
    print("3rd arg: max value for bar e.g. : 4095")

ser = serial.Serial(sys.argv[1])

print sys.argv
arg = sys.argv[1:]

bar = []
bar.append(Bar(title=arg[1].ljust(10), max_value=int(arg[2]), fallback=True))
bar[0].cursor.clear_lines(len(arg) / 2 + 3)
bar[0].cursor.save()

title = []
title.append(arg[1])

for i in range(3, len(arg) - 1, 2):
    bar.append(
        Bar(title=arg[i].ljust(10), max_value=int(arg[i + 1]), fallback=True))
    title.append(arg[i])
    #print("title: " + arg[i] + " max: " + arg[i+1])

val = []
for i in range(len(arg) / 2):
    val.append(0)
def simul_css(CataSect, _CssImg, cssbands, filtnumb, npi):

    # print('Process'+str(npi)

    CssHei, CssWid = _CssImg.shape

    OutSecStr = ''

    LenCatSec = len(CataSect)
    procedi = 0

    if IfProgBarOn == True:
        bar = Bar(max_value=LenCatSec,
                  empty_color=7,
                  filled_color=18 + npi * 6,
                  title='Process-' + str(npi))
        bar.cursor.clear_lines(1)
        bar.cursor.save()

    for procedi, cataline in enumerate(CataSect, 1):

        # if ((float(cataline['MOD_NUV_css'])<0) or (float(cataline['MOD_WNUV_css'])<0) or (float(cataline['MOD_NUV_css'])>50)):
        #     continue

        np.random.seed()

        ident = str(cataline['IDENT'])
        if DebugTF == True:
            print('\n', ident,
                  '\n--------------------------------------------')

        objwind = csstpkg.windcut(_CssImg, cataline, StampSize)
        # print(objwind)

        if objwind is None:
            if DebugTF == True:
                print('\033[31mError: ' + 'Object stamp cutting error.\033[0m')
            continue
        # csstpkg.DataArr2Fits(objwind.data, ident+'_convwin.fits')

        objwinshape = objwind.shape
        # objwind.data = objwind.data * ExpCssFrm

        WinImgBands = np.zeros(
            (len(cssbands), objwinshape[0],
             objwinshape[1]))  # 3-D array contains images of all the cssbands

        if DebugTF == True:
            if IfPlotObjWin == True:
                csstpkg.PlotObjWin(objwind, cataline)
            print(' '.join(
                ['RA DEC:',
                 str(cataline['RA']),
                 str(cataline['DEC'])]))

        outcatrowi = [ident, cataline['Z_BEST']]

        # Photometry for the central object on the convolved window
        ObjWinPhot_DeBkg = csstpkg.CentrlPhot(objwind.data,
                                              id=str(outcatrowi[0]) +
                                              " ConvWdW DeBkg")
        ObjWinPhot_DeBkg.Bkg(idb=str(outcatrowi[0]) + " ConvWdW DeBkg",
                             debug=DebugTF,
                             thresh=1.5,
                             minarea=10,
                             deblend_nthresh=32,
                             deblend_cont=0.01)
        ObjWinPhot_DeBkg.Centract(idt=str(outcatrowi[0]) + " ConvWdW DeBkg",
                                  thresh=2.5,
                                  minarea=10,
                                  deblend_nthresh=32,
                                  deblend_cont=0.1,
                                  debug=DebugTF,
                                  sub_backgrd_bool=True)
        if ObjWinPhot_DeBkg.centobj is np.nan:
            if DebugTF == True:
                print('--- No central object detected in convolved image ---')
            continue
        else:
            ObjWinPhot_DeBkg.KronR(idk=str(outcatrowi[0]) + " ConvWdW",
                                   debug=DebugTF,
                                   mask_bool=True)

        NeConv_DeBkg, ErrNeConv_DeBkg = ObjWinPhot_DeBkg.EllPhot(
            ObjWinPhot_DeBkg.kronr, mask_bool=True)

        if ((NeConv_DeBkg <= 0) or (NeConv_DeBkg is np.nan)):
            if DebugTF == True:
                print(
                    'NeConv_DeBkg for a winimg <= 0 or NeConv_DeBkg is np.nan')
            continue

        noisebkg_conv = ObjWinPhot_DeBkg.bkgstd

        if DebugTF == True:
            print('self.bkg Flux & ErrFlux =', ObjWinPhot_DeBkg.bkgmean,
                  ObjWinPhot_DeBkg.bkgstd)
            print('Class processed NeConv_DeBkg & ErrNeConv_DeBkg:',
                  NeConv_DeBkg, ErrNeConv_DeBkg)

        # Read model SED to NDArray
        # modsednum = cataline['MOD_BEST']
        sedname = seddir + 'Id' + '{:0>9}'.format(ident) + '.spec'
        modsed, readflag = csstpkg.readsed(sedname)
        if readflag == 1:
            modsed[:, 1] = csstpkg.magab2flam(
                modsed[:, 1], modsed[:, 0]
            )  # to convert model SED from magnitude to f_lambda(/A)
        else:
            print('model sed not found.')
            continue
        bandi = 0
        NeBands = []
        magsimorigs = {}
        magsims = {}
        scalings = []
        ObjWinPhot_DeBkg_Errs = []

        for cssband, numb in zip(cssbands, filtnumb):

            expcss = 150. * numb  # s
            # cssbandpath = thrghdir+cssband+'.txt'
            magsim = csstpkg.Sed2Mag(modsed, cssband, MagSim_Zero[cssband])
            magsims[cssband] = magsim
            lambpivot = csstpkg.pivot(cssband)
            if DebugTF == False:
                flambandarr = 1
            elif DebugTF == True:
                if cssband == 'i4':
                    cssband = 'i'
                elif cssband == 'uB':
                    cssband = 'u'
                elif cssband == 'gN':
                    cssband = 'g'
                flambandmod = csstpkg.magab2flam(
                    float(cataline['MOD_' + cssband + '_css']), lambpivot)
                flambandsim = csstpkg.magab2flam(magsim, lambpivot)
                flambandarr = np.array([[lambpivot, flambandmod],
                                        [lambpivot, flambandsim]])
            NeABandObs = csstpkg.NeFromSED(modsed,
                                           cssband,
                                           expcss,
                                           TelArea,
                                           flambandarr,
                                           debug=DebugTF)

            # # magaband0 = csstpkg.Ne2MagAB(NeABandObs, cssband, expcss, TelArea)
            # delmag = float(cataline['MOD_' + cssband + '_css']) - magsim
            # NeABand = NeABandObs*10**(-0.4*delmag)  # in e-/band/exptime/telarea
            # NeBands.append(NeABand)

            NeBands.append(NeABandObs)
            NeABand = NeABandObs
            NeABand = np.random.poisson(lam=NeABandObs)  # Do poisson randomize

            if DebugTF == True:
                print(' Mag from Sim for ' + cssband + ' band =', magsim)
                # print(' Mag from Ne Calculation =', magaband0)
                # print('  DeltaMag_'+cssband+' = ', float(cataline['MOD_' + cssband + '_css'])-magsim, delmag)
                print(' '.join([
                    'Counts on ConvImg:',
                    str(NeConv_DeBkg / ExpCssFrm), 'e-'
                ]))
                print(' '.join(
                    [cssband, 'band model electrons = ',
                     str(NeABand), 'e-']))
                print('MOD_' + cssband + '_css =',
                      cataline['MOD_' + cssband + '_css'])
                if NeABand > 0:
                    magsimorigs[cssband] = csstpkg.Ne2MagAB(
                        NeABand, cssband, expcss, TelArea)
                else:
                    magsimorigs[cssband] = 99
                print('Magsim_' + cssband + ' =', magsimorigs[cssband])

            Scl2Sed = NeABand / NeConv_DeBkg  # To scale stamp HST detection/s to SED in e-/band/exptime/telarea.
            scalings.append(Scl2Sed)

            if DebugTF == True:
                print('Scaling Factor: ', Scl2Sed)

            # ZeroLevel = config.getfloat('Hst2Css', 'BZero')
            SkyLevel = csstpkg.backsky[cssband] * expcss
            DarkLevel = config.getfloat('Hst2Css', 'BDark') * expcss
            RNCssFrm = config.getfloat('Hst2Css', 'RNCss')

            # IdealImg = objwind.data * Scl2Sed + SkyLevel + DarkLevel  # e-
            IdealImg = ObjWinPhot_DeBkg.data_bkg * Scl2Sed  # + SkyLevel + DarkLevel  # e-
            ObjWinPhot_DeBkg_Errs.append(ObjWinPhot_DeBkg.bkgstd * Scl2Sed)

            BkgNoiseTot = np.sqrt(SkyLevel + DarkLevel + RNCssFrm**2 * numb)
            if BkgNoiseTot > noisebkg_conv * Scl2Sed:
                Noise2Add = np.sqrt(BkgNoiseTot**2 -
                                    (noisebkg_conv * Scl2Sed)**2)
            else:
                Noise2Add = 0

            if DebugTF == True:
                print('Noise Total ' + cssband + ' band: ', BkgNoiseTot)
                print('Noise Stamp ' + cssband + ' band: ',
                      noisebkg_conv * Scl2Sed)
                print('Noise Added ' + cssband + ' band: ', Noise2Add)

            # ImgPoiss = copy.deepcopy(IdealImg)
            # ImgPoiss[ImgPoiss>0] = np.random.poisson(lam=IdealImg[IdealImg>0]*ExpCssFrm, size=IdealImg[IdealImg>0].shape)/ExpCssFrm
            NoisNormImg = csstpkg.NoiseArr(objwinshape,
                                           loc=0,
                                           scale=Noise2Add,
                                           func='normal')

            # DigitizeImg = IdealImg/Gain
            DigitizeImg = np.round(
                (IdealImg + NoisNormImg) /
                Gain)  # IdealImg have already been poissonized
            # DigitizeImg = np.round((ImgPoiss + NoisNormImg + ZeroLevel) / Gain)

            # if DebugTF == True:
            #     csstpkg.DataArr2Fits(DigitizeImg, 'ImgWinSim_Gain_RN_'+ident+'_'+cssband+'.fits')

            WinImgBands[bandi, ::] = DigitizeImg

            bandi = bandi + 1

        if DebugTF == True:
            print('Stack all bands and detect objects:')

        WinImgStack = WinImgBands[2:7, ::].sum(0)
        # WinImgStack = WinImgBands[0:7,::].sum(0)
        # print(WinImgStack.shape)
        # AduStack, ErrAduStack, ObjectStack, KronRStack, MaskStack = septract(WinImgStack, id=str(outcatrowi[0])+" Stack", debug=DebugTF, thresh=1.2, minarea=10)
        StackPhot = csstpkg.CentrlPhot(WinImgStack, id=ident + " Stack")
        StackPhot.Bkg(idb=ident + " Stack",
                      debug=DebugTF,
                      thresh=1.5,
                      minarea=10)
        StackPhot.Centract(idt=ident + " Stack",
                           thresh=1.5,
                           minarea=10,
                           deblend_nthresh=32,
                           deblend_cont=0.1,
                           debug=DebugTF)
        if StackPhot.centobj is np.nan:
            if DebugTF == True:
                print('No central object on STACK image.')
            continue
        else:
            A_stack = StackPhot.centobj['a']
            B_stack = StackPhot.centobj['b']
            Drms_stack = np.sqrt(
                (A_stack**2 + B_stack**2) / 2) * pixscale  # RMS size in arcsec
            StackPhot.KronR(idk=ident + " Stack",
                            debug=DebugTF,
                            mask_bool=True)
        AduStack, ErrAduStack = StackPhot.EllPhot(StackPhot.kronr,
                                                  mask_bool=True)
        if AduStack is np.nan:
            if DebugTF == True:
                print('RSS error for STACK image.')
            continue

        if DebugTF == True:
            csstpkg.PlotKronrs(WinImgStack, StackPhot)

        bandi = 0
        for cssband, numb in zip(cssbands, filtnumb):
            expcss = 150. * numb  # s
            if DebugTF == True:
                plt.hist(
                    WinImgBands[bandi, ::].flatten(),
                    bins=np.arange(30) - 15,
                )
                plt.title(' '.join([cssband, 'simul image']))
                plt.show()

            AduObser, ErrAduObs, npix, bkgrms = csstpkg.septractSameAp(
                WinImgBands[bandi, ::],
                StackPhot,
                StackPhot.centobj,
                StackPhot.kronr,
                mask_det=StackPhot.mask_other,
                debug=DebugTF,
                annot=cssband + '_cssos',
                thresh=1.2,
                minarea=10,
                sub_backgrd_bool=False)
            # print(scalings)
            ErrAduTot = np.sqrt(ErrAduObs**2 + npix *
                                (noisebkg_conv * scalings[bandi])**2)
            # ErrAduTot = np.sqrt(ErrAduObs**2+npix*ObjWinPhot_DeBkg_Errs[bandi]**2)

            if AduObser > 0:
                SNR = AduObser / ErrAduTot
                # FluxMsr = csstpkg.Ne2Fnu(AduObser*Gain,cssband,expcss,TelArea)
                FluxMsr = AduObser * fluxadu_zeros[bandi]
                FLuxErr = ErrAduTot * fluxadu_zeros[bandi]  # FluxMsr/SNR

            else:
                # FluxMsr = 0
                # FLuxErr = csstpkg.Ne2Fnu(ErrAduTot*Gain,cssband,expcss,TelArea)
                FluxMsr = AduObser * fluxadu_zeros[bandi]
                FLuxErr = ErrAduTot * fluxadu_zeros[bandi]
                SNR = 0

            if DebugTF == True:
                npixel = math.pi * (
                    ObjWinPhot_DeBkg.centobj['a'] * csstpkg.kphotpar *
                    ObjWinPhot_DeBkg.kronr) * (ObjWinPhot_DeBkg.centobj['b'] *
                                               csstpkg.kphotpar *
                                               ObjWinPhot_DeBkg.kronr)
                print(' '.join(
                    [cssband, 'band model e- =',
                     str(NeBands[bandi]), 'e-']))
                print(' '.join([
                    cssband, 'band simul e- =',
                    str(AduObser * Gain), 'e-', ' ErrNe=',
                    str(ErrAduTot * Gain)
                ]))
                # print(AduObser, Gain, NeBands[bandi], -2.5*math.log10(AduObser*Gain/NeBands[bandi]))
                print('SNR =', AduObser / ErrAduTot)
                print('Npixel =', npixel)
                # print(' '.join([cssband, 'band mag_model = ', str(cataline['MOD_' + cssband + '_css']), '(AB mag)']))
                # print(' '.join([cssband, 'band Magsim_orig = ', str(magsimorigs[bandi]), '(AB mag)']))
                # print(' '.join([cssband, 'band Mag_simul = ', str(MagObser), '(AB mag)']))
                # print(' '.join([cssband, 'band magerr_simul = ', str(ErrMagObs), '(AB mag)']))
                # print(' '.join(['Magsim - Magsimorig =', str(MagObser-magsimorigs[bandi])]))

            # if cssband=='uB':
            #     modmag = cataline['MOD_u_css']
            # elif cssband == 'gN':
            #     modmag = cataline['MOD_g_css']
            # elif cssband == 'i4':
            #     modmag = cataline['MOD_i_css']
            # else:
            #     modmag = cataline['MOD_' + cssband + '_css']
            outcatrowi = outcatrowi + [magsims[cssband], FluxMsr, FLuxErr, SNR]
            bandi = bandi + 1

        del WinImgBands

        outcatrowi = outcatrowi + [npix, Drms_stack]
        # outcatrowi = outcatrowi + [Drms_stack]
        colnumb = len(outcatrowi)

        OutRowStr = ('{} ' +
                     (colnumb - 1) * '{:15.6E}').format(*outcatrowi) + '\n'
        OutSecStr = OutSecStr + OutRowStr

        if IfProgBarOn == True:
            bar.cursor.restore()  # Return cursor to start
            bar.draw(value=procedi)

    if IfProgBarOn == True:
        bar.cursor.restore()  # Return cursor to start
        bar.draw(value=bar.max_value)  # Draw the bar!

    # OutCatSecQueue.put(OutSecStr)
    # _FinishQueue.put(1)

    # write_lock.acquire()
    with write_lock:
        OutCssCat.write(OutSecStr)
        OutCssCat.flush()
    # write_lock.release()

    print('\n')
Exemple #25
0
def get_progressive_bar(total_count):
    from progressive.bar import Bar
    bar = Bar(max_value=total_count, fallback=True)
    bar.cursor.clear_lines(2)
    bar.cursor.save()
    return bar
Exemple #26
0
def output_age_matrix():
    from progressive.bar import Bar
    word_vectors = pickle.load(open('./parameters_200.bin', 'rb'))
    all_data_x = []
    all_data_y = []
    index = 0
    window_size = 2
    word_vector_size = 200
    text_vector_size = window_size * word_vector_size
    image_vector_size = word_vector_size
    #进度条相关参数
    total_count = users.count()
    bar = Bar(max_value=total_count, fallback=True)
    bar.cursor.clear_lines(2)
    bar.cursor.save()
    finish_count = 0
    for user in users.find({'got_image_descriptions': True}):
        #age=ages[uid]
        #根据用户年龄过滤
        #if age>=100 or age<=5:
        #    continue
        correct_status = 0
        #根据合法status数量过滤
        for status in user['statuses']:
            if is_not_good_status(status):
                continue
            else:
                correct_status += 1
        if correct_status < 50:
            continue
        length = []
        # text_vector=[numpy.zeros(text_vector_size)]
        # time_vector=[numpy.zeros(time_vector_size)]
        # image_vector=[numpy.zeros(image_vector_size)]
        # for description in user['information']['descriptions']:
        #     for word in description:
        #         try:
        #             image_vector.append(word_vectors[word])
        #         except:
        #             continue
        # if len(image_vector)==1:
        #     continue
        text = []
        for status in user['statuses']:
            if is_not_good_status(status):
                continue
            for word in status['text']:
                try:
                    text.append(word_vectors[word])
                except Exception as e:
                    continue
            if len(text) > 100:
                continue
            #sentence_vector=[]
            #for word in status['text']:
            #    try:
            #        sentence_vector.append(list(word_vectors[word]))
            #    except:
            #        continue
            #for i in range(0,len(sentence_vector)-window_size):
            #    text.append(get_text_convolution(sentence_vector[i:i+window_size]))
            # created_at=status['time']
            # try:
            #     hour=get_hour(created_at)
            #     time_vector.append(time_vectors[hour])
            # except Exception as e:
            #     continue
            #     pass
            # length.append(len(status['text']))
        # text_vector_mean=numpy.mean(text_vector,axis=0)
        # if len(text)<50:
        #     continue
        text = text[0:100]
        text_vector = numpy.array(text)  #numpy.max(text_vector,axis=0)
        #text_vector=numpy.max(text,axis=0)
        text_vector = text_vector.reshape(
            (text_vector.shape[0] * text_vector.shape[1]))
        # time_vector=numpy.sum(time_vector,axis=0)
        #data.append(get_age_class(age))
        if user['information']['gender'] == 'm':
            all_data_y.append(1)
        else:
            all_data_y.append(0)
        # data=data+list(text_vector_mean)+list(time_vector)+list(numpy.max(image_vector,axis=0))+[numpy.max(length),numpy.min(length),numpy.mean(length),len(length)]
        #data=data+list(time_vector)+[numpy.max(length),numpy.min(length),numpy.mean(length),len(length)]
        #data=data+list(text_vector)
        all_data_x.append(text_vector)
        #all_data.append(data)
        index += 1
        finish_count += 1
        bar.cursor.restore()
        bar.draw(value=finish_count)
        #if finish_count>500:
        #    break
    all_data_x = numpy.array(all_data_x)
    all_data_y = numpy.array(all_data_y)
    train_set_x = all_data_x[0:index * 3 / 4]
    train_set_y = all_data_y[0:index * 3 / 4]
    train_set = (train_set_x, train_set_y)
    valid_set_x = all_data_x[index * 3 / 4:index * 7 / 8]
    valid_set_y = all_data_y[index * 3 / 4:index * 7 / 8]
    valid_set = (valid_set_x, valid_set_y)
    test_set_x = all_data_x[index * 7 / 8:]
    test_set_y = all_data_y[index * 7 / 8:]
    test_set = (test_set_x, test_set_y)
    pickle.dump((train_set, valid_set, test_set),
                open('gender_matrix_with_time.data', 'wb'))
Exemple #27
0
#判断 39 40 41是否都因为drop t_error_maker停止
@dec_progressive
def get_error_status():
	while True:
		Last_SQL_Error_39 = get_slave_statue(conn39,'show slave status;')['Last_SQL_Error']
		Last_SQL_Error_40 = get_slave_statue(conn40,'show slave status;')['Last_SQL_Error']
		Last_SQL_Error_41 = get_slave_statue(conn41,'show slave status;')['Last_SQL_Error']
		if Last_SQL_Error_39 == Last_SQL_Error_40 == Last_SQL_Error_41 == error_message:
			break
		else:
			time.sleep(1)

if __name__ == '__main__':
	MAX_VALUE = 100
	bar = Bar(max_value=MAX_VALUE, fallback=True)
	bar.cursor.clear_lines(2)
	bar.cursor.save()
	i=0

	#不显示MySQL的warning
	filterwarnings('ignore',category=pymysql.Warning)
	#连接3306 制造复制异常函数
	print(u"连接3306 制造复制异常函数")
	error_maker(host='172.16.65.36', port=3306, user='******',password='******',db='fandb',charset='utf8')
	conn39 = get_conn('10.0.1.39',3306,'root','mysql')
	conn40 = get_conn('10.0.1.40',3306,'root','mysql')
	conn41 = get_conn('10.0.1.41',3306,'root','mysql')

	#判断 39 40 41是否都因为drop t_error_maker停止
	print(u"判断 39 40 41是否都因为drop t_error_maker停止")
Exemple #28
0
def process_request(obj):
 # check if year is mentioned
 if obj.year:
  _year = obj.year
 else:
  _year = datetime.datetime.now().strftime("%Y")

 if obj.month:
  _month = obj.month
 else:
  _month = datetime.datetime.now().strftime("%B")

 if obj.day:
  _day = obj.day
 else:
  _day = datetime.datetime.now().strftime("%d")

 _id = None 
 if obj.id:
  _id = obj.id

 _amount = None
 if obj.amount:
  _amount = obj.amount

 _tag = None
 if obj.tag:
  _tag = obj.tag

 # create a database connection
 path = os.path.join(os.path.dirname(__file__,), '..', '..', 'data', _year+'.db')
 conn = sqlite3.connect(path)

 # create object of Bar
 prog = Bar(filled_color=2, title=u'Updating record , please wait ....')

 # create cursor object and clear lines
 prog.cursor.clear_lines(2)

 # save the state of the cursor
 prog.cursor.save()

 for i in range(101):
  time.sleep(0.02)
  prog.cursor.restore()
  prog.draw(i)
 

 # check if _id is present ,if yes delete that particular id record
 if _id:
  if _amount:
   cursor = conn.execute("UPDATE '%s' SET AMOUNT='%f' WHERE DAY='%d' AND ID='%d'" %(_month, float(_amount),int(_day), int(_id)))
  if _tag:
   cursor = conn.execute("UPDATE '%s' SET TAG='%s' WHERE DAY='%d' AND ID='%d'" %(_month, _tag,int(_day), int(_id)))
 else:
 #display records 
  if _amount:
   cursor = conn.execute("UPDATE '%s' SET AMOUNT='%f' WHERE DAY='%d'" %(_month, float(_amount),int(_day)))
  if _tag:
   cursor = conn.execute("UPDATE '%s' SET TAG='%s' WHERE DAY='%d'" %(_month, _tag ,int(_day)))
 conn.commit()
 conn.close()
def simul_css(CataSect, _CssImg, cssbands, filtnumb, npi):

    # print('Process'+str(npi)

    CssHei, CssWid = _CssImg.shape

    OutSecStr = ''

    LenCatSec = len(CataSect)
    procedi = 0

    if IfProgBarOn == True:
        bar = Bar(max_value=LenCatSec,
                  empty_color=7,
                  filled_color=18 + npi * 6,
                  title='Process-' + str(npi))
        bar.cursor.clear_lines(1)
        bar.cursor.save()

    for procedi, cataline in enumerate(CataSect, 1):

        np.random.seed()

        ident = str(cataline['IDENT'])

        objwind = csstpkg.windcut(_CssImg, cataline)

        if objwind is None:
            if DebugTF == True:
                print('--- Object window cutting error ---')
            continue
        # DataArr2Fits(objwind, ident+'_convwin.fits')
        objwinshape = objwind.shape
        objwind.data = objwind.data * ExpCssFrm

        WinImgBands = np.zeros(
            (len(cssbands), objwinshape[0],
             objwinshape[1]))  # 3-D array contains images of all the cssbands

        if IfPlotObjWin == True:
            csstpkg.PlotObjWin(objwind, cataline)

        outcatrowi = [ident, cataline['Z_BEST']]

        if DebugTF == True:
            print(' '.join([
                ident, '\nRA DEC:',
                str(cataline['RA']),
                str(cataline['DEC'])
            ]))

        # Photometry for the central object on the convolved window
        ObjWinPhot_DeBkg = csstpkg.CentrlPhot(objwind.data,
                                              id=str(outcatrowi[0]) +
                                              " ConvWdW DeBkg")
        ObjWinPhot_DeBkg.Bkg(idb=str(outcatrowi[0]) + " ConvWdW DeBkg",
                             debug=DebugTF,
                             thresh=1.5,
                             minarea=10,
                             deblend_nthresh=32,
                             deblend_cont=0.01)
        ObjWinPhot_DeBkg.Centract(idt=str(outcatrowi[0]) + " ConvWdW DeBkg",
                                  thresh=2.5,
                                  deblend_nthresh=32,
                                  deblend_cont=0.1,
                                  minarea=10,
                                  debug=DebugTF,
                                  sub_backgrd_bool=True)
        if ObjWinPhot_DeBkg.centobj is np.nan:
            if DebugTF == True:
                print('--- No central object detected in convolved image ---')
            continue
        else:
            ObjWinPhot_DeBkg.KronR(idk=str(outcatrowi[0]) + " ConvWdW",
                                   debug=DebugTF,
                                   mask_bool=True)

        NeConv_DeBkg, ErrNeConv_DeBkg = ObjWinPhot_DeBkg.EllPhot(
            ObjWinPhot_DeBkg.kronr, mask_bool=True)

        if ((NeConv_DeBkg <= 0) or (NeConv_DeBkg is np.nan)):
            if DebugTF == True:
                print(
                    'NeConv_DeBkg for a winimg <= 0 or NeConv_DeBkg is np.nan')
            continue

        noisebkg_conv = ObjWinPhot_DeBkg.bkg.background_rms_median

        if DebugTF == True:
            print('self.bkg Flux & ErrFlux =',
                  ObjWinPhot_DeBkg.bkg.background_median,
                  ObjWinPhot_DeBkg.bkg.background_rms_median)
            print('Class processed NeConv_DeBkg & ErrNeConv_DeBkg:',
                  NeConv_DeBkg, ErrNeConv_DeBkg)

        # Read model SED to NDArray
        # modsednum = cataline['MOD_BEST']
        sedname = seddir + 'Id' + '{:0>9}'.format(ident) + '.spec'
        modsed, readflag = csstpkg.readsed(sedname)
        if readflag == 1:
            modsed[:, 1] = csstpkg.mag2flam(
                modsed[:, 1], modsed[:, 0]
            )  # to convert model SED from magnitude to f_lambda(/A)
        else:
            print('model sed not found.')
            continue
        bandi = 0
        NeBands = []
        magsimorigs = []
        scalings = []

        for cssband, numb in zip(cssbands, filtnumb):

            expcss = 150. * numb  # s
            # cssbandpath = thrghdir+cssband+'.txt'

            NeABand0 = csstpkg.NeObser(modsed, cssband, expcss,
                                       TelArea)  # *cataline['SCALE_BEST']
            if NeABand0 < 1:
                continue
            magaband0 = csstpkg.Ne2MagAB(NeABand0, cssband, expcss, TelArea)
            delmag = magaband0 - float(cataline['MOD_' + cssband + '_css'])
            magsimorig_band = magaband0 - delmag
            NeABand = NeABand0 / 10**(-0.4 * delmag)

            # print(NeABand0, magaband0, delmag)

            NeBands.append(NeABand)

            if DebugTF == True:
                print(' '.join(
                    [cssband, 'band model electrons = ',
                     str(NeABand), 'e-']))
                print('MOD_' + cssband + '_css =',
                      cataline['MOD_' + cssband + '_css'])
                magsimorigs.append(
                    csstpkg.Ne2MagAB(NeABand, cssband, expcss, TelArea))
                print('Magsim_' + cssband + ' =', magsimorigs[bandi])

            Scl2Sed = NeABand / NeConv_DeBkg
            scalings.append(Scl2Sed)

            if DebugTF == True:
                print(ident, 'Scaling Factor: ', Scl2Sed)

            # ZeroLevel = config.getfloat('Hst2Css', 'BZero')
            SkyLevel = 0  #csstpkg.backsky[cssband] * expcss
            DarkLevel = 0  #config.getfloat('Hst2Css', 'BDark') * expcss
            RNCssFrm = config.getfloat('Hst2Css', 'RNCss')

            # IdealImg = objwind.data * Scl2Sed + SkyLevel + DarkLevel  # e-
            IdealImg = ObjWinPhot_DeBkg.data_bkg * Scl2Sed  # + SkyLevel + DarkLevel  # e-
            # IdealImg[IdealImg < 0] = 0

            if DebugTF == True:
                # csstpkg.DataArr2Fits(IdealImg/Gain, 'Ideal_Zero_Gain_check_'+ident+'_'+cssband+'.fits')

                # Testing photometry for the scaled convolved window's central object
                ObjWinPhot = csstpkg.CentrlPhot(IdealImg,
                                                id=(ident + " SclTesting"))
                try:
                    ObjWinPhot.Bkg(idb=ident + " SclTesting",
                                   debug=DebugTF,
                                   thresh=1.5,
                                   minarea=10,
                                   deblend_nthresh=32,
                                   deblend_cont=0.01)
                except Exception as e:
                    # print(NeConv_DeBkg, NeABand, IdealImg)
                    continue
                ObjWinPhot.Centract(idt=ident + " SclTesting",
                                    thresh=2.5,
                                    deblend_nthresh=32,
                                    deblend_cont=0.1,
                                    minarea=10,
                                    debug=DebugTF,
                                    sub_backgrd_bool=False)
                if ObjWinPhot.centobj is np.nan:
                    print(
                        '--- No central object detected in testing photometry image---'
                    )
                    continue
                else:
                    ObjWinPhot.KronR(idk=ident + " SclTesting",
                                     debug=DebugTF,
                                     mask_bool=True)

                NeConv, ErrNeConv = ObjWinPhot.EllPhot(ObjWinPhot.kronr,
                                                       mask_bool=True)

                print(' '.join([
                    'Model electrons:',
                    str(NeABand), '\nTesting Photometry After scaling:',
                    str(NeConv)
                ]))

            BkgNoiseTot = (SkyLevel + DarkLevel + RNCssFrm**2 * numb)**0.5
            if BkgNoiseTot > noisebkg_conv * Scl2Sed:
                Noise2Add = (BkgNoiseTot**2 -
                             (noisebkg_conv * Scl2Sed)**2)**0.5
            else:
                Noise2Add = 0

            if DebugTF == True:
                print('Added Noise ' + cssband + ' band: ', Noise2Add)

            # ImgPoiss = np.random.poisson(lam=IdealImg, size=objwinshape)
            ImgPoiss = IdealImg
            NoisNormImg = csstpkg.NoiseArr(objwinshape,
                                           loc=0,
                                           scale=Noise2Add,
                                           func='normal')

            # DigitizeImg = np.round((ImgPoiss + NoisNorm + ZeroLevel) / Gain)
            DigitizeImg = (ImgPoiss + NoisNormImg) / Gain
            # DigitizeImg = IdealImg/Gain

            if DebugTF == True:
                csstpkg.DataArr2Fits(
                    DigitizeImg, 'Ideal_Zero_Gain_RN_check_' + ident + '_' +
                    cssband + '.fits')

            WinImgBands[bandi, ::] = DigitizeImg

            bandi = bandi + 1

        if DebugTF == True:
            print('Stack all bands and detect objects:')

        WinImgStack = WinImgBands.sum(0)
        # print(WinImgStack.shape)
        # AduStack, ErrAduStack, ObjectStack, KronRStack, MaskStack = septract(WinImgStack, id=str(outcatrowi[0])+" Stack", debug=DebugTF, thresh=1.2, minarea=10)
        StackPhot = csstpkg.CentrlPhot(WinImgStack, id=ident + " Stack")
        StackPhot.Bkg(idb=ident + " Stack",
                      debug=DebugTF,
                      thresh=1.5,
                      minarea=10)
        StackPhot.Centract(idt=ident + " Stack",
                           thresh=1.5,
                           minarea=10,
                           deblend_nthresh=32,
                           deblend_cont=0.1,
                           debug=DebugTF)
        if StackPhot.centobj is np.nan:
            if DebugTF == True:
                print('No central object on STACK image.')
            continue
        else:
            StackPhot.KronR(idk=ident + " Stack",
                            debug=DebugTF,
                            mask_bool=True)
        AduStack, ErrAduStack = StackPhot.EllPhot(StackPhot.kronr,
                                                  mask_bool=True)
        if AduStack is np.nan:
            if DebugTF == True:
                print('RSS error for STACK image.')
            continue

        if DebugTF == True:
            csstpkg.PlotKronrs(WinImgStack, StackPhot)

        bandi = 0
        for cssband, numb in zip(cssbands, filtnumb):
            expcss = 150. * numb  # s
            if DebugTF == True:
                plt.hist(
                    WinImgBands[bandi, ::].flatten(),
                    bins=np.arange(30) - 15,
                )
                plt.title(' '.join([cssband, 'simul image']))
                plt.show()

            SameApObj = csstpkg.CentrlPhot(WinImgBands[bandi, ::],
                                           id=ident + ' ' + cssband +
                                           ' band CentralExtract')
            SameApObj.Bkg(idb=ident + ' ' + cssband + ' band CentralExtract',
                          debug=DebugTF,
                          thresh=1.5,
                          minarea=10)
            SameApObj.Centract(idt=ident + ' ' + cssband +
                               ' band CentralExtract',
                               thresh=1.2,
                               minarea=10,
                               deblend_nthresh=32,
                               deblend_cont=0.1,
                               debug=DebugTF,
                               sub_backgrd_bool=False)
            if SameApObj.centobj is np.nan:
                if DebugTF == True:
                    print('No central object on simulated image.')
                SNR = -99
                MagObser = -99
                ErrMagObs = -99
                # AduObser, ErrAduObs = csstpkg.septractSameAp(WinImgBands[bandi, ::], StackPhot, ObjWinPhot_DeBkg.centobj, ObjWinPhot_DeBkg.kronr, mask_det=StackPhot.mask_other, debug=DebugTF, annot=cssband+'_cssos', thresh=1.2, minarea=10, sub_backgrd_bool=False)
                # if AduObser>0:
                #     MagObser = -2.5 * math.log10(AduObser) + magab_zeros[bandi]
                #     ErrMagObs = -1
                #     ErrAduTot = (ErrAduObs ** 2 + (noisebkg_conv * scalings[bandi]) ** 2) ** 0.5
                #     SNR = AduObser / ErrAduTot
                # else:
                #     SNR = -99
                #     MagObser = -99
                #     ErrMagObs = -99
            else:
                # AduObser, ErrAduObs = csstpkg.septractSameAp(WinImgBands[bandi, ::], StackPhot, ObjWinPhot_DeBkg.centobj, ObjWinPhot_DeBkg.kronr, mask_det=StackPhot.mask_other, debug=DebugTF, annot=cssband+'_cssos', thresh=1.2, minarea=10, sub_backgrd_bool=False)

                AduObser, ErrAduObs = csstpkg.septractSameAp(
                    WinImgBands[bandi, ::],
                    StackPhot,
                    StackPhot.centobj,
                    StackPhot.kronr,
                    mask_det=StackPhot.mask_other,
                    debug=DebugTF,
                    annot=cssband + '_cssos',
                    thresh=1.2,
                    minarea=10,
                    sub_backgrd_bool=False)

                if AduObser > 0:
                    ErrAduTot = (ErrAduObs**2 +
                                 (noisebkg_conv * scalings[bandi])**2)**0.5
                    SNR = AduObser / ErrAduTot
                    # MagObser = Ne2MagAB(AduObser*Gain,cssband,expcss,TelArea)
                    MagObser = -2.5 * math.log10(AduObser) + magab_zeros[bandi]
                    ErrMagObs = 2.5 * math.log10(1 + 1 / SNR)
                    if DebugTF == True:
                        if ((cssband == 'r') &
                            (np.abs(MagObser -
                                    cataline['MOD_' + cssband + '_css']) > 1)):
                            csstpkg.DataArr2Fits(objwind.data,
                                                 ident + '_convwin_r.fits')
                            csstpkg.DataArr2Fits(WinImgStack,
                                                 ident + '_stack.fits')
                else:
                    SNR = -99
                    MagObser = -99
                    ErrMagObs = -99

                if DebugTF == True:
                    npixel = math.pi * (
                        ObjWinPhot_DeBkg.centobj['a'] * csstpkg.kphotpar *
                        ObjWinPhot_DeBkg.kronr) * (
                            ObjWinPhot_DeBkg.centobj['b'] * csstpkg.kphotpar *
                            ObjWinPhot_DeBkg.kronr)
                    print(' '.join([
                        cssband, 'band model e- =',
                        str(NeBands[bandi]), 'e-'
                    ]))
                    print(' '.join([
                        cssband, 'band simul e- =',
                        str(AduObser * Gain), 'e-', ' ErrNe=',
                        str(ErrAduTot * Gain)
                    ]))
                    # print(AduObser, Gain, NeBands[bandi], -2.5*math.log10(AduObser*Gain/NeBands[bandi]))
                    print('SNR =', AduObser / ErrAduTot)
                    print('Npixel =', npixel)
                    print(' '.join([
                        cssband, 'band mag_model = ',
                        str(cataline['MOD_' + cssband + '_css']), '(AB mag)'
                    ]))
                    print(' '.join([
                        cssband, 'band Magsim_orig = ',
                        str(magsimorigs[bandi]), '(AB mag)'
                    ]))
                    print(' '.join([
                        cssband, 'band Mag_simul = ',
                        str(MagObser), '(AB mag)'
                    ]))
                    print(' '.join([
                        cssband, 'band magerr_simul = ',
                        str(ErrMagObs), '(AB mag)'
                    ]))
                    print(' '.join([
                        'Magsim - Magsimorig =',
                        str(MagObser - magsimorigs[bandi])
                    ]))

            outcatrowi = outcatrowi + [
                cataline['MOD_' + cssband + '_css'], MagObser, ErrMagObs, SNR
            ]
            bandi = bandi + 1

        del WinImgBands
        colnumb = len(outcatrowi)

        OutRowStr = ('{} ' +
                     (colnumb - 1) * '{:8.3f}').format(*outcatrowi) + '\n'
        OutSecStr = OutSecStr + OutRowStr

        if IfProgBarOn == True:
            bar.cursor.restore()  # Return cursor to start
            bar.draw(value=procedi)

    if IfProgBarOn == True:
        bar.cursor.restore()  # Return cursor to start
        bar.draw(value=bar.max_value)  # Draw the bar!

    # OutCatSecQueue.put(OutSecStr)
    # _FinishQueue.put(1)

    # write_lock.acquire()
    with write_lock:
        OutCssCat.write(OutSecStr)
        OutCssCat.flush()
    # write_lock.release()

    print('\n')
Exemple #30
0
def simul_css(CataSect, cssbands, filtnumb, npi):

    # print('Process'+str(npi)

    OutSecStr = ''

    LenCatSec = len(CataSect)
    procedi = 0

    if IfProgBarOn == True:
        bar = Bar(max_value=LenCatSec,
                  empty_color=7,
                  filled_color=18 + npi * 6,
                  title='Process-' + str(npi))
        bar.cursor.clear_lines(1)
        bar.cursor.save()

    for procedi, cataline in enumerate(CataSect, 1):

        np.random.seed()

        ident = str(cataline['IDENT'])

        outcatrowi = [ident, cataline['Z_BEST']]

        if DebugTF == True:
            print(' '.join([
                ident, '\nRA DEC:',
                str(cataline['RA']),
                str(cataline['DEC'])
            ]))

        sedname = seddir + 'Id' + '{:0>9}'.format(ident) + '.spec'
        modsed = csstpkg.readsed(sedname)
        modsed[:, 1] = csstpkg.mag2flam(
            modsed[:, 1],
            modsed[:,
                   0])  # to convert model SED from magnitude to f_lambda(/A)

        for bandi, cssband in enumerate(cssbands):

            mag_en = csstpkg.mag_ener(modsed, cssband, magab_ener_zeros[bandi])
            mag_ph = csstpkg.mag_phot(modsed, cssband, magab_phot_zeros[bandi])
            if DebugTF == True:
                # print(' '.join([cssband, 'band model electrons = ', str(NeABand), 'e-']))
                print('MOD_' + cssband + '_css =',
                      cataline['MOD_' + cssband + '_css'])
                print('Magsim_' + cssband + '_photon =', mag_ph)
                print('Magsim_' + cssband + '_energy =', mag_en)

            outcatrowi = outcatrowi + [
                cataline['MOD_' + cssband + '_css'], mag_en, mag_ph
            ]
            # bandi = bandi + 1

        # del WinImgBands
        colnumb = len(outcatrowi)

        OutRowStr = ('{} ' +
                     (colnumb - 1) * '{:8.3f}').format(*outcatrowi) + '\n'
        OutSecStr = OutSecStr + OutRowStr

        if IfProgBarOn == True:
            bar.cursor.restore()  # Return cursor to start
            bar.draw(value=procedi)

    if IfProgBarOn == True:
        bar.cursor.restore()  # Return cursor to start
        bar.draw(value=bar.max_value)  # Draw the bar!

    with write_lock:
        OutCssCat.write(OutSecStr)
        OutCssCat.flush()
    # write_lock.release()

    print('\n')
Exemple #31
0
'''convert img to hdf5
'''
import glob
import pandas as pd
from keras.preprocessing import image
import numpy as np
from progressive.bar import Bar

imglist = glob.glob('data/train-jpg/*.jpg')
allimg = sorted(imglist, key=lambda x: int(x[21:].split('.')[0]))

imgsize = 224
totalimg = len(imglist)
alldata = np.zeros([totalimg, imgsize * imgsize * 3], dtype=np.uint8)

pb = Bar(max_value=totalimg, fallback=True)
pb.cursor.clear_lines(2)
pb.cursor.save()
for index, img in enumerate(allimg):
    img = image.load_img(img, target_size=(224, 224))
    alldata[index, :] = image.img_to_array(img).reshape(-1)
    pb.cursor.restore()
    pb.draw(index)

pddata = pd.DataFrame(alldata)
pddata.to_hdf('data/train_amazon.hdf5', 'train_amazon')
Exemple #32
0
    data.batch_size_test(1000)

    lr = 1e-3
    gamma = 1
    beta_1 = 0.9
    beta_2 = 0.999
    total_epoch = 100

    loss_cache = 10
    for epoch in xrange(1, total_epoch + 1):
        print 'Epoch: {}/{}'.format(epoch, total_epoch)

        # Training (Mini-batch)
        now = time.time()
        data.shuffle()
        bar = Bar(max_value=n)
        bar.cursor.clear_lines(2)  # Make some room
        bar.cursor.save()  # Mark starting line
        for _ in xrange(data.batch_run()):
            net.input(data.next_batch())
            net.forward()
            net.backward(lr, beta_1, beta_2, epoch)
            bar.cursor.restore()  # Return cursor to start
            bar.draw(value=data.get_count())
        t = time.time() - now
        acc, loss = net.get_record()
        loss_avg = np.array(loss).mean()
        loss_diff = loss_avg - loss_cache
        loss_cache = loss_avg
        print 'Acc:    ', np.array(acc).mean()
        print 'Loss:   ', loss_avg