def __init__(self, total_p):
self.cursor = 1
self.total = total_p
self.bar = Bar(max_value=total_p)
self.bar.cursor.clear_lines(10)
self.bar.cursor.save()
self.bar.draw(value=self.cursor)
def _progress_loop(self, request_method, url, body):
response = request_method(url, data=json.dumps(body))
body = response.json()
max_tasks = body["num_total_tasks"]
bar = Bar(max_value=max_tasks, title="Completed Tasks",
num_rep="percentage", filled_color=2)
n = ProgressTree(term=self.t)
n.cursor.clear_lines(self.t.height - 1)
while True:
response = request_method(url, data=json.dumps(body))
body = response.json()
n.cursor.restore()
n.cursor.clear_lines(self.t.height - 1)
n.cursor.save()
bar.draw(value=body["num_finished_tasks"])
presp = body
presp["energy_history"] = sorted(presp["energy_history"])[:10]
del presp["best_location"]
print(json.dumps(presp, indent=4))
if response.status_code == 200:
return response
sleep(2.0)
def check_mail(step=3, host=None, username=None): # check emails per `step` minutes
pymail = mail(host=host, username=username)
info = pymail.mails_info()
sleep_time = step/100
#sleep_time = 0.02 # for test
try:
while 1:
bar = Bar(max_value=10, fallback=True, filled_color=1, title='Check Emails...')
bar.cursor.clear_lines(2)
bar.cursor.save()
for i in range(11):
sleep(sleep_time)
# We restore the cursor to saved position before writing
bar.cursor.restore()
# Now we draw the bar
bar.draw(value=i)
if pymail.mails_info()[0] >= info[0]:
info = pymail.mails_info()
win = tk.Tk()
win.title("Mail Checking")
win.resizable(False, False)
win.update()
scrn_width, scrn_height = win.maxsize()
win.geometry('200x70+%d+%d'%((scrn_width-200)/2,(scrn_height-65)/2))
warn_gui(win)
playsound()
win.mainloop()
print('\033[0;32mChecked...\033[0m')
pymail.quit()
return
except(KeyboardInterrupt, SystemExit):
pymail.quit()
def process_request(obj):
# check if year is mentioned
if obj.year:
_year = obj.year
else:
_year = datetime.datetime.now().strftime("%Y")
if obj.month:
_month = obj.month.title()
else:
_month = datetime.datetime.now().strftime("%B")
if obj.day:
_day = obj.day
else:
_day = datetime.datetime.now().strftime("%d")
# create a database connection
path = os.path.join(os.path.dirname(__file__, ), '..', '..', 'data',
_year + '.db')
conn = sqlite3.connect(path)
# create object of Bar
prog = Bar(filled_color=2, title=u'Fetching details , please wait ....')
# create cursor object and clear lines
prog.cursor.clear_lines(2)
# save the state of the cursor
prog.cursor.save()
for i in range(101):
time.sleep(0.02)
prog.cursor.restore()
prog.draw(i)
if obj.day:
#display records
cursor = conn.execute("SELECT * FROM '%s' WHERE DAY='%d'" %
(_month, int(_day)))
else:
cursor = conn.execute("SELECT * FROM '%s'" % _month)
if cursor:
print "Successully fetched details"
else:
print "No details to fetch"
sys.exit()
# displaying results in nice tabular format [TODO]
headers = ['id', 'day', 'amount', 'tag']
data = []
for row in cursor:
data.append(row)
# displaying the table
print tabulate(data, headers, tablefmt="fancy_grid")
# graphical representations [TODO]
# close the database connection
conn.close()
def __init__(self, title='Progress', max_value=100):
try:
from progressive.bar import Bar
self.support_progressive = True
self.bar = Bar(max_value=max_value, title='Receiving')
self.bar.cursor.clear_lines(1)
self.bar.cursor.save()
except ImportError:
self.support_progressive = False
from progress.bar import Bar
self.bar = Bar('Receiving', max=max_value)
def _draw(self, tree, indent=0):
"""Recurse through ``tree`` and draw all nodes"""
if all([isinstance(tree, dict), type(tree) != BarDescriptor]):
for k, v in sorted(tree.items()):
bar_desc, subdict = v[0], v[1]
args = [self.cursor.term] + bar_desc.get("args", [])
kwargs = dict(title_pos="above", indent=indent, title=k)
kwargs.update(bar_desc.get("kwargs", {}))
b = Bar(*args, **kwargs)
b.draw(value=bar_desc["value"].value, flush=False)
self._draw(subdict, indent=indent + self.indent)
def process_request(obj):
# check if year is mentioned
if obj.year:
_year = obj.year
else:
_year = datetime.datetime.now().strftime("%Y")
if obj.month:
_month = obj.month.title()
else:
_month = datetime.datetime.now().strftime("%B")
# create a database connection
path = os.path.join(os.path.dirname(__file__, ), '..', '..', 'data',
_year + '.db')
conn = sqlite3.connect(path)
# create object of Bar
prog = Bar(filled_color=2, title=u'Fetching details , please wait ....')
# create cursor object and clear lines
prog.cursor.clear_lines(2)
# save the state of the cursor
prog.cursor.save()
for i in range(101):
time.sleep(0.02)
prog.cursor.restore()
prog.draw(i)
# fetching month records
cursor = conn.execute("SELECT * FROM '%s'" % _month)
if cursor:
print "Successully fetched details"
else:
print "No details to fetch"
sys.exit()
_total = 0
# displaying results in nice tabular format [TODO]
for row in cursor:
_total += row[2]
print "total expenses of month {} = {}".format(_month, str(_total))
# graphical representations [TODO]
# close the database connection
conn.close()
def SearchByKeyword(keyword):
print '=>Search by keyword: %s' % keyword
client = MongoClient('localhost', 27017)
db = client['YFCC100M']
collection = db['dataset_with_tag']
query = {'$and': [
{'User_tags': {'$in': [keyword]}},
{'Longitude': {'$ne': ''}},
{'Latitude': {'$ne': ''}}
]}
fields = {}
fields['User_tags'] = True
fields['Longitude'] = True
fields['Latitude'] = True
cursor = collection.find(query, fields)
total = cursor.count()
print "There is %d documents about %s." % (total, keyword)
print
print '=>Runing point_in_polygon:'
bbox_dict = Get_area_list()
print
user_tags_list = []
locations_list = []
now = 0 # progress
bar = Bar(max_value=total)
bar.cursor.clear_lines(2)
bar.cursor.save()
for doc in cursor:
lon = doc['Longitude']
lat = doc['Latitude']
user_tags = doc['User_tags']
country = point_in_polygon_with_shapely(bbox_dict, lon, lat)
now += 1
bar.cursor.restore()
bar.draw(value=now)
if country != None:
user_tags_list.append(user_tags)
locations_list.append(country)
return user_tags_list, locations_list
class prog_bar(object):
def __init__(self, total_p):
self.cursor = 1
self.total = total_p
self.bar = Bar(max_value=total_p)
self.bar.cursor.clear_lines(10)
self.bar.cursor.save()
self.bar.draw(value=self.cursor)
def new_page(self, cnt):
self.cursor += cnt
self.bar.cursor.restore()
self.bar.draw(value=self.cursor)
def reflash(self, time, size, in_q, wait_q, out_q, urlc):
self.bar.cursor.restore()
self.bar.draw(value=self.cursor)
print "-------------"
print "done:" + str(size) + " "
print "in_q:" + str(in_q) + " "
print "wait_q:" + str(wait_q) + " "
print "out_q:" + str(out_q) + " "
print "spent: " + str(int(time / 60)) + " min" + " "
print "rest: " + str(int(time / size * urlc / 60)) + " min"
def reflash_r(self, f_cnt, q_size, c_cnt, curren_f, filename, e_cnt):
self.bar.cursor.restore()
self.bar.draw(value=self.cursor)
print "-------------"
print "fetch:" + str(f_cnt) + " "
print "q_size:" + str(q_size) + " "
print "commit: " + str(c_cnt) + " "
print "curren_f: " + str(curren_f) + " "
print "file_size: " + str(os.path.getsize(filename)) + " "
print "error_count: " + str(e_cnt) + " "
def get_stat(self, done, time, filename, wait_q, e_cnt):
f_size = os.path.getsize(filename)
self.bar.cursor.restore()
self.bar.draw(value=self.cursor)
print "done: " + str(done) + " file_size: " + str(int(f_size))
print "total_size: " + str(
int(f_size / self.cursor * (self.total - self.cursor)))
print "spent: " + str(int(time / 60)) + " min"
print "rest: " + str(
int(time / self.cursor * (self.total - self.cursor) / 60)) + " min"
print "wait_url: " + str(wait_q)
print "error_count: " + str(e_cnt)
class prog_bar(object):
def __init__(self, total_p):
self.cursor=1
self.total=total_p
self.bar = Bar(max_value=total_p)
self.bar.cursor.clear_lines(10)
self.bar.cursor.save()
self.bar.draw(value=self.cursor)
def new_page(self,cnt):
self.cursor+=cnt
self.bar.cursor.restore()
self.bar.draw(value=self.cursor)
def reflash(self,time,size,in_q,wait_q,out_q,urlc):
self.bar.cursor.restore()
self.bar.draw(value=self.cursor)
print "-------------"
print "done:"+str(size)+" "
print "in_q:"+str(in_q)+" "
print "wait_q:"+str(wait_q)+" "
print "out_q:"+str(out_q)+" "
print "spent: "+str(int(time/60))+" min"+" "
print "rest: "+str(int(time/size*urlc/60))+" min"
def reflash_r(self,f_cnt,q_size,c_cnt,curren_f,filename,e_cnt):
self.bar.cursor.restore()
self.bar.draw(value=self.cursor)
print "-------------"
print "fetch:"+str(f_cnt)+" "
print "q_size:"+str(q_size)+" "
print "commit: "+str(c_cnt)+" "
print "curren_f: "+str(curren_f)+" "
print "file_size: "+str(os.path.getsize(filename))+" "
print "error_count: "+str(e_cnt)+" "
def get_stat(self,done,time,filename,wait_q,e_cnt):
f_size=os.path.getsize(filename)
self.bar.cursor.restore()
self.bar.draw(value=self.cursor)
print "done: "+str(done)+" file_size: "+str(int(f_size))
print "total_size: "+str(int(f_size/self.cursor*(self.total-self.cursor)))
print "spent: "+str(int(time/60))+" min"
print "rest: "+str(int(time/self.cursor*(self.total-self.cursor)/60))+" min"
print "wait_url: "+str(wait_q)
print "error_count: "+str(e_cnt)
def start_attack():
import MySQLdb
global avalible_pass
Log.log_info('Starting attack...')
bar = Bar(max_value=len(user_name_list) * len(pass_list))
bar.cursor.clear_lines(2)
bar.cursor.save()
i = 0
for user_name in user_name_list:
for password in pass_list:
try:
access = MySQLdb.connect(host=host,
port=port,
user=user_name,
passwd=password)
database_access = access.cursor()
# print 'username:'******'Access denied for user' in str(exception):
pass
# Log.log_error('Access denied')
else:
Log.log_error(str(exception))
bar.cursor.clear_lines(2)
sys.exit()
finally:
def __init__(self, total_p):
self.cursor=1
self.total=total_p
self.bar = Bar(max_value=total_p)
self.bar.cursor.clear_lines(10)
self.bar.cursor.save()
self.bar.draw(value=self.cursor)
def _draw(self, tree, indent=0):
"""Recurse through ``tree`` and draw all nodes"""
if all([
isinstance(tree, dict),
type(tree) != BarDescriptor
]):
for k, v in sorted(tree.items()):
bar_desc, subdict = v[0], v[1]
args = [self.cursor.term] + bar_desc.get("args", [])
kwargs = dict(title_pos="above", indent=indent, title=k)
kwargs.update(bar_desc.get("kwargs", {}))
b = Bar(*args, **kwargs)
b.draw(value=bar_desc["value"].value, flush=False)
self._draw(subdict, indent=indent + self.indent)
def get_bar(title, max_value):
bar = self._bar if hasattr(self, '_bar') else None
if bar == None or bar.title != title:
bar = Bar(max_value=max_value, title=title)
bar.cursor.clear_lines(2) # Make some room
bar.cursor.save() # Mark starting line
self._bar = bar
return bar
def do_progress(self, args):
"""show progressbar"""
if self.hydra_running():
pass
else:
print('[x] MiniHydra is not running!')
return
try:
bar = Bar(max_value=self._hydra.get_total_size())
bar.cursor.clear_lines(2)
bar.cursor.save()
while True:
sleep(0.1)
bar.cursor.restore()
bar.draw(value=self._hydra.get_current_pos())
except KeyboardInterrupt:
pass
def main():
print('Start time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))
stockid.deleteContent('yield.txt')
stockid.gen_stockid()
f = open('stockid.txt', 'r')
result = list()
line_count = id_count('stockid.txt')
y = open('yield.txt', 'a')
y.write(" ID Price Avg. SD\n")
count = 0
bar = Bar(max_value=line_count, fallback=True)
bar.cursor.clear_lines(2)
bar.cursor.save()
for line in f.readlines():
line = line.strip()
y = open('yield.txt', 'a')
# check the yield rate is >= 6.25%
stock_yield = parse_stock.get_average_dividend(int(line))
stock_price = parse_stock.get_current_price(int(line))
historical_price = parse_stock.historical_price(int(line))
recent_PER = parse_stock.get_recent_PER(int(line))
total_volume = parse_stock.get_current_volume(int(line))
time.sleep(0.2)
if (stock_price == None or historical_price == 0.0):
#y.write(line + str(' *****error***** ') + stock_price + str(' historical_price:') + str(historical_price) + str('\n'))
continue
if (stock_yield / float(stock_price) >= 0.0625 and # rule 1
float(stock_price) / historical_price <= 0.6 and # rule 2
float(stock_price) >= 10 and # rule 3
0 < float(stock_price) / recent_PER <= 15 and # rule 4
int(total_volume) >= 50): # rule 5
y.write(line + str(' ') + stock_price + str(' ') +
str(stock_yield) + str('\n'))
count = count + 1
bar.cursor.restore()
bar.draw(value=count)
print('Finish time: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))
def simple():
"""Simple example using just the Bar class
This example is intended to show usage of the Bar class at the lowest
level.
"""
MAX_VALUE = 100
# Create our test progress bar
bar = Bar(max_value=MAX_VALUE, fallback=True)
bar.cursor.clear_lines(2)
# Before beginning to draw our bars, we save the position
# of our cursor so we can restore back to this position before writing
# the next time.
bar.cursor.save()
for i in range(MAX_VALUE + 1):
sleep(0.1 * random.random())
# We restore the cursor to saved position before writing
bar.cursor.restore()
# Now we draw the bar
bar.draw(value=i)
def simple():
"""Simple example using just the Bar class
This example is intended to show usage of the Bar class at the lowest
level.
"""
MAX_VALUE = 100
# Create our test progress bar
bar = Bar(max_value=MAX_VALUE, fallback=True)
bar.cursor.clear_lines(2)
# Before beginning to draw our bars, we save the position
# of our cursor so we can restore back to this position before writing
# the next time.
bar.cursor.save()
for i in range(MAX_VALUE + 1):
sleep(0.1 * random.random())
# We restore the cursor to saved position before writing
bar.cursor.restore()
# Now we draw the bar
bar.draw(value=i)
def bar(response, chunk_size=8192, filename="Seicmic data"):
if response.headers['Content-Length'] == None:
print("Downloading data to %s......" % filename)
chunk_all = response.read()
if type(chunk_all) != str:
itype = 'bytes'
else:
itype = 'str'
# print("No response of server")
# sys.exit(1)
else:
total_size = response.headers['Content-Length'].strip()
total_size = int(total_size)
cycle = int(total_size/chunk_size)+1
bytes_so_far = 0
MAX_VALUE = 100
bar = Bar(max_value=MAX_VALUE, num_rep="percentage", fallback=True)
bar.cursor.clear_lines(2)
bar.cursor.save()
for i in range(cycle):
chunk = response.read(chunk_size)
if i == 0:
if type(chunk) != str:
chunk_all = b''
itype = 'bytes'
else:
chunk_all = ''
itype = 'str'
bytes_so_far += len(chunk)
chunk_all += chunk
percentage = int(bytes_so_far*100/total_size)
bar.cursor.restore()
bar.draw(value=percentage)
print('%dbyte/%dbyte' %(bytes_so_far,total_size))
return chunk_all, itype
class ProgressBar(object):
def __init__(self, title='Progress', max_value=100):
try:
from progressive.bar import Bar
self.support_progressive = True
self.bar = Bar(max_value=max_value, title='Receiving')
self.bar.cursor.clear_lines(1)
self.bar.cursor.save()
except ImportError:
self.support_progressive = False
from progress.bar import Bar
self.bar = Bar('Receiving', max=max_value)
def next(self, count=0):
if self.support_progressive:
self.bar.cursor.restore()
self.bar.draw(value=count)
else:
self.bar.next()
def finish(self):
if not self.support_progressive:
self.bar.finish()
#!/usr/bin/env python3
# -*- encoding: utf8 -*-
# Example: python -m scripts.glyph_model_convert FILE1 FILE2
import sys
import json
from importlib import import_module
from progressive.bar import Bar
sys.path.append('src/cv')
GlyphModel = import_module('glyph_model').GlyphModel
with open(sys.argv[1], 'r') as in_fp, open(sys.argv[2], 'w') as out_fp:
file_js = json.load(in_fp)
result = {}
bar = Bar(max_value=len(file_js), fallback=True)
bar.cursor.clear_lines(2)
bar.cursor.save()
i = 0
for char in file_js:
glyph = file_js[char]
try:
glyph_model = GlyphModel(glyph)
result[char] = glyph_model.get_model_object()
except Exception as e:
print('In file "' + sys.argv[1] + '", gid "' + char + '":')
print(e)
i += 1
bar.cursor.restore()
bar.draw(value=i)
json.dump(result, out_fp)
expcss = 150. * numb # s
magab_zeros.append(
csstpkg.MagAB_Zero(Gain, cssband, expcss, TelArea))
print('Filter scheme flag:', schemecode)
namelists = map(lambda mag, magerr, snr, aband: \
[mag + aband, magerr+aband, snr+aband], \
['Magsim_'] * len(cssbands), ['ErrMag_'] * len(cssbands), ['SNR_'] * len(cssbands), cssbands)
colnames = ['ID', 'Z_BEST'] + list(itertools.chain(*namelists))
OutCat = Table(names=colnames)
# print(OutCat)
NColOut = len(OutCat.colnames)
if IfProgBarOn == True:
bar = Bar(max_value=len(CatOfTile), empty_color=7, filled_color=4)
bar.cursor.clear_lines(2) # Make some room
bar.cursor.save() # Mark starting line
for i in range(len(CatOfTile)):
# if DebugTF == True:
# print(CatOfTile[i]['IDENT'])
a = CatOfTile[i]['a_image_css']
b = CatOfTile[i]['b_image_css']
theta = CatOfTile[i]['theta_image']
ident = str(CatOfTile[i]['IDENT'])
# Cut a window of the object as objwind
# cutwidrad = int((a*math.cos(theta/180.*math.pi)+b*abs(math.sin(theta/180.*math.pi)))*5)
# cutheirad = int((a*abs(math.sin(theta/180.*math.pi))+b*math.cos(theta/180.*math.pi))*5)
import serial
from blessings import Terminal
from progressive.bar import Bar
if len(sys.argv) == 1:
print("1st arg: serial port e.g. : /dev/tty. device ")
print("2nd arg: parse string e.g. : \'$A1:\'")
print("3rd arg: max value for bar e.g. : 4095")
ser = serial.Serial(sys.argv[1])
print sys.argv
arg = sys.argv[1:]
bar = []
bar.append(Bar(title=arg[1].ljust(10), max_value=int(arg[2]), fallback=True))
bar[0].cursor.clear_lines(len(arg) / 2 + 3)
bar[0].cursor.save()
title = []
title.append(arg[1])
for i in range(3, len(arg) - 1, 2):
bar.append(
Bar(title=arg[i].ljust(10), max_value=int(arg[i + 1]), fallback=True))
title.append(arg[i])
#print("title: " + arg[i] + " max: " + arg[i+1])
val = []
for i in range(len(arg) / 2):
val.append(0)
def simul_css(CataSect, _CssImg, cssbands, filtnumb, npi):
# print('Process'+str(npi)
CssHei, CssWid = _CssImg.shape
OutSecStr = ''
LenCatSec = len(CataSect)
procedi = 0
if IfProgBarOn == True:
bar = Bar(max_value=LenCatSec,
empty_color=7,
filled_color=18 + npi * 6,
title='Process-' + str(npi))
bar.cursor.clear_lines(1)
bar.cursor.save()
for procedi, cataline in enumerate(CataSect, 1):
# if ((float(cataline['MOD_NUV_css'])<0) or (float(cataline['MOD_WNUV_css'])<0) or (float(cataline['MOD_NUV_css'])>50)):
# continue
np.random.seed()
ident = str(cataline['IDENT'])
if DebugTF == True:
print('\n', ident,
'\n--------------------------------------------')
objwind = csstpkg.windcut(_CssImg, cataline, StampSize)
# print(objwind)
if objwind is None:
if DebugTF == True:
print('\033[31mError: ' + 'Object stamp cutting error.\033[0m')
continue
# csstpkg.DataArr2Fits(objwind.data, ident+'_convwin.fits')
objwinshape = objwind.shape
# objwind.data = objwind.data * ExpCssFrm
WinImgBands = np.zeros(
(len(cssbands), objwinshape[0],
objwinshape[1])) # 3-D array contains images of all the cssbands
if DebugTF == True:
if IfPlotObjWin == True:
csstpkg.PlotObjWin(objwind, cataline)
print(' '.join(
['RA DEC:',
str(cataline['RA']),
str(cataline['DEC'])]))
outcatrowi = [ident, cataline['Z_BEST']]
# Photometry for the central object on the convolved window
ObjWinPhot_DeBkg = csstpkg.CentrlPhot(objwind.data,
id=str(outcatrowi[0]) +
" ConvWdW DeBkg")
ObjWinPhot_DeBkg.Bkg(idb=str(outcatrowi[0]) + " ConvWdW DeBkg",
debug=DebugTF,
thresh=1.5,
minarea=10,
deblend_nthresh=32,
deblend_cont=0.01)
ObjWinPhot_DeBkg.Centract(idt=str(outcatrowi[0]) + " ConvWdW DeBkg",
thresh=2.5,
minarea=10,
deblend_nthresh=32,
deblend_cont=0.1,
debug=DebugTF,
sub_backgrd_bool=True)
if ObjWinPhot_DeBkg.centobj is np.nan:
if DebugTF == True:
print('--- No central object detected in convolved image ---')
continue
else:
ObjWinPhot_DeBkg.KronR(idk=str(outcatrowi[0]) + " ConvWdW",
debug=DebugTF,
mask_bool=True)
NeConv_DeBkg, ErrNeConv_DeBkg = ObjWinPhot_DeBkg.EllPhot(
ObjWinPhot_DeBkg.kronr, mask_bool=True)
if ((NeConv_DeBkg <= 0) or (NeConv_DeBkg is np.nan)):
if DebugTF == True:
print(
'NeConv_DeBkg for a winimg <= 0 or NeConv_DeBkg is np.nan')
continue
noisebkg_conv = ObjWinPhot_DeBkg.bkgstd
if DebugTF == True:
print('self.bkg Flux & ErrFlux =', ObjWinPhot_DeBkg.bkgmean,
ObjWinPhot_DeBkg.bkgstd)
print('Class processed NeConv_DeBkg & ErrNeConv_DeBkg:',
NeConv_DeBkg, ErrNeConv_DeBkg)
# Read model SED to NDArray
# modsednum = cataline['MOD_BEST']
sedname = seddir + 'Id' + '{:0>9}'.format(ident) + '.spec'
modsed, readflag = csstpkg.readsed(sedname)
if readflag == 1:
modsed[:, 1] = csstpkg.magab2flam(
modsed[:, 1], modsed[:, 0]
) # to convert model SED from magnitude to f_lambda(/A)
else:
print('model sed not found.')
continue
bandi = 0
NeBands = []
magsimorigs = {}
magsims = {}
scalings = []
ObjWinPhot_DeBkg_Errs = []
for cssband, numb in zip(cssbands, filtnumb):
expcss = 150. * numb # s
# cssbandpath = thrghdir+cssband+'.txt'
magsim = csstpkg.Sed2Mag(modsed, cssband, MagSim_Zero[cssband])
magsims[cssband] = magsim
lambpivot = csstpkg.pivot(cssband)
if DebugTF == False:
flambandarr = 1
elif DebugTF == True:
if cssband == 'i4':
cssband = 'i'
elif cssband == 'uB':
cssband = 'u'
elif cssband == 'gN':
cssband = 'g'
flambandmod = csstpkg.magab2flam(
float(cataline['MOD_' + cssband + '_css']), lambpivot)
flambandsim = csstpkg.magab2flam(magsim, lambpivot)
flambandarr = np.array([[lambpivot, flambandmod],
[lambpivot, flambandsim]])
NeABandObs = csstpkg.NeFromSED(modsed,
cssband,
expcss,
TelArea,
flambandarr,
debug=DebugTF)
# # magaband0 = csstpkg.Ne2MagAB(NeABandObs, cssband, expcss, TelArea)
# delmag = float(cataline['MOD_' + cssband + '_css']) - magsim
# NeABand = NeABandObs*10**(-0.4*delmag) # in e-/band/exptime/telarea
# NeBands.append(NeABand)
NeBands.append(NeABandObs)
NeABand = NeABandObs
NeABand = np.random.poisson(lam=NeABandObs) # Do poisson randomize
if DebugTF == True:
print(' Mag from Sim for ' + cssband + ' band =', magsim)
# print(' Mag from Ne Calculation =', magaband0)
# print(' DeltaMag_'+cssband+' = ', float(cataline['MOD_' + cssband + '_css'])-magsim, delmag)
print(' '.join([
'Counts on ConvImg:',
str(NeConv_DeBkg / ExpCssFrm), 'e-'
]))
print(' '.join(
[cssband, 'band model electrons = ',
str(NeABand), 'e-']))
print('MOD_' + cssband + '_css =',
cataline['MOD_' + cssband + '_css'])
if NeABand > 0:
magsimorigs[cssband] = csstpkg.Ne2MagAB(
NeABand, cssband, expcss, TelArea)
else:
magsimorigs[cssband] = 99
print('Magsim_' + cssband + ' =', magsimorigs[cssband])
Scl2Sed = NeABand / NeConv_DeBkg # To scale stamp HST detection/s to SED in e-/band/exptime/telarea.
scalings.append(Scl2Sed)
if DebugTF == True:
print('Scaling Factor: ', Scl2Sed)
# ZeroLevel = config.getfloat('Hst2Css', 'BZero')
SkyLevel = csstpkg.backsky[cssband] * expcss
DarkLevel = config.getfloat('Hst2Css', 'BDark') * expcss
RNCssFrm = config.getfloat('Hst2Css', 'RNCss')
# IdealImg = objwind.data * Scl2Sed + SkyLevel + DarkLevel # e-
IdealImg = ObjWinPhot_DeBkg.data_bkg * Scl2Sed # + SkyLevel + DarkLevel # e-
ObjWinPhot_DeBkg_Errs.append(ObjWinPhot_DeBkg.bkgstd * Scl2Sed)
BkgNoiseTot = np.sqrt(SkyLevel + DarkLevel + RNCssFrm**2 * numb)
if BkgNoiseTot > noisebkg_conv * Scl2Sed:
Noise2Add = np.sqrt(BkgNoiseTot**2 -
(noisebkg_conv * Scl2Sed)**2)
else:
Noise2Add = 0
if DebugTF == True:
print('Noise Total ' + cssband + ' band: ', BkgNoiseTot)
print('Noise Stamp ' + cssband + ' band: ',
noisebkg_conv * Scl2Sed)
print('Noise Added ' + cssband + ' band: ', Noise2Add)
# ImgPoiss = copy.deepcopy(IdealImg)
# ImgPoiss[ImgPoiss>0] = np.random.poisson(lam=IdealImg[IdealImg>0]*ExpCssFrm, size=IdealImg[IdealImg>0].shape)/ExpCssFrm
NoisNormImg = csstpkg.NoiseArr(objwinshape,
loc=0,
scale=Noise2Add,
func='normal')
# DigitizeImg = IdealImg/Gain
DigitizeImg = np.round(
(IdealImg + NoisNormImg) /
Gain) # IdealImg have already been poissonized
# DigitizeImg = np.round((ImgPoiss + NoisNormImg + ZeroLevel) / Gain)
# if DebugTF == True:
# csstpkg.DataArr2Fits(DigitizeImg, 'ImgWinSim_Gain_RN_'+ident+'_'+cssband+'.fits')
WinImgBands[bandi, ::] = DigitizeImg
bandi = bandi + 1
if DebugTF == True:
print('Stack all bands and detect objects:')
WinImgStack = WinImgBands[2:7, ::].sum(0)
# WinImgStack = WinImgBands[0:7,::].sum(0)
# print(WinImgStack.shape)
# AduStack, ErrAduStack, ObjectStack, KronRStack, MaskStack = septract(WinImgStack, id=str(outcatrowi[0])+" Stack", debug=DebugTF, thresh=1.2, minarea=10)
StackPhot = csstpkg.CentrlPhot(WinImgStack, id=ident + " Stack")
StackPhot.Bkg(idb=ident + " Stack",
debug=DebugTF,
thresh=1.5,
minarea=10)
StackPhot.Centract(idt=ident + " Stack",
thresh=1.5,
minarea=10,
deblend_nthresh=32,
deblend_cont=0.1,
debug=DebugTF)
if StackPhot.centobj is np.nan:
if DebugTF == True:
print('No central object on STACK image.')
continue
else:
A_stack = StackPhot.centobj['a']
B_stack = StackPhot.centobj['b']
Drms_stack = np.sqrt(
(A_stack**2 + B_stack**2) / 2) * pixscale # RMS size in arcsec
StackPhot.KronR(idk=ident + " Stack",
debug=DebugTF,
mask_bool=True)
AduStack, ErrAduStack = StackPhot.EllPhot(StackPhot.kronr,
mask_bool=True)
if AduStack is np.nan:
if DebugTF == True:
print('RSS error for STACK image.')
continue
if DebugTF == True:
csstpkg.PlotKronrs(WinImgStack, StackPhot)
bandi = 0
for cssband, numb in zip(cssbands, filtnumb):
expcss = 150. * numb # s
if DebugTF == True:
plt.hist(
WinImgBands[bandi, ::].flatten(),
bins=np.arange(30) - 15,
)
plt.title(' '.join([cssband, 'simul image']))
plt.show()
AduObser, ErrAduObs, npix, bkgrms = csstpkg.septractSameAp(
WinImgBands[bandi, ::],
StackPhot,
StackPhot.centobj,
StackPhot.kronr,
mask_det=StackPhot.mask_other,
debug=DebugTF,
annot=cssband + '_cssos',
thresh=1.2,
minarea=10,
sub_backgrd_bool=False)
# print(scalings)
ErrAduTot = np.sqrt(ErrAduObs**2 + npix *
(noisebkg_conv * scalings[bandi])**2)
# ErrAduTot = np.sqrt(ErrAduObs**2+npix*ObjWinPhot_DeBkg_Errs[bandi]**2)
if AduObser > 0:
SNR = AduObser / ErrAduTot
# FluxMsr = csstpkg.Ne2Fnu(AduObser*Gain,cssband,expcss,TelArea)
FluxMsr = AduObser * fluxadu_zeros[bandi]
FLuxErr = ErrAduTot * fluxadu_zeros[bandi] # FluxMsr/SNR
else:
# FluxMsr = 0
# FLuxErr = csstpkg.Ne2Fnu(ErrAduTot*Gain,cssband,expcss,TelArea)
FluxMsr = AduObser * fluxadu_zeros[bandi]
FLuxErr = ErrAduTot * fluxadu_zeros[bandi]
SNR = 0
if DebugTF == True:
npixel = math.pi * (
ObjWinPhot_DeBkg.centobj['a'] * csstpkg.kphotpar *
ObjWinPhot_DeBkg.kronr) * (ObjWinPhot_DeBkg.centobj['b'] *
csstpkg.kphotpar *
ObjWinPhot_DeBkg.kronr)
print(' '.join(
[cssband, 'band model e- =',
str(NeBands[bandi]), 'e-']))
print(' '.join([
cssband, 'band simul e- =',
str(AduObser * Gain), 'e-', ' ErrNe=',
str(ErrAduTot * Gain)
]))
# print(AduObser, Gain, NeBands[bandi], -2.5*math.log10(AduObser*Gain/NeBands[bandi]))
print('SNR =', AduObser / ErrAduTot)
print('Npixel =', npixel)
# print(' '.join([cssband, 'band mag_model = ', str(cataline['MOD_' + cssband + '_css']), '(AB mag)']))
# print(' '.join([cssband, 'band Magsim_orig = ', str(magsimorigs[bandi]), '(AB mag)']))
# print(' '.join([cssband, 'band Mag_simul = ', str(MagObser), '(AB mag)']))
# print(' '.join([cssband, 'band magerr_simul = ', str(ErrMagObs), '(AB mag)']))
# print(' '.join(['Magsim - Magsimorig =', str(MagObser-magsimorigs[bandi])]))
# if cssband=='uB':
# modmag = cataline['MOD_u_css']
# elif cssband == 'gN':
# modmag = cataline['MOD_g_css']
# elif cssband == 'i4':
# modmag = cataline['MOD_i_css']
# else:
# modmag = cataline['MOD_' + cssband + '_css']
outcatrowi = outcatrowi + [magsims[cssband], FluxMsr, FLuxErr, SNR]
bandi = bandi + 1
del WinImgBands
outcatrowi = outcatrowi + [npix, Drms_stack]
# outcatrowi = outcatrowi + [Drms_stack]
colnumb = len(outcatrowi)
OutRowStr = ('{} ' +
(colnumb - 1) * '{:15.6E}').format(*outcatrowi) + '\n'
OutSecStr = OutSecStr + OutRowStr
if IfProgBarOn == True:
bar.cursor.restore() # Return cursor to start
bar.draw(value=procedi)
if IfProgBarOn == True:
bar.cursor.restore() # Return cursor to start
bar.draw(value=bar.max_value) # Draw the bar!
# OutCatSecQueue.put(OutSecStr)
# _FinishQueue.put(1)
# write_lock.acquire()
with write_lock:
OutCssCat.write(OutSecStr)
OutCssCat.flush()
# write_lock.release()
print('\n')
def get_progressive_bar(total_count):
from progressive.bar import Bar
bar = Bar(max_value=total_count, fallback=True)
bar.cursor.clear_lines(2)
bar.cursor.save()
return bar
def output_age_matrix():
from progressive.bar import Bar
word_vectors = pickle.load(open('./parameters_200.bin', 'rb'))
all_data_x = []
all_data_y = []
index = 0
window_size = 2
word_vector_size = 200
text_vector_size = window_size * word_vector_size
image_vector_size = word_vector_size
#进度条相关参数
total_count = users.count()
bar = Bar(max_value=total_count, fallback=True)
bar.cursor.clear_lines(2)
bar.cursor.save()
finish_count = 0
for user in users.find({'got_image_descriptions': True}):
#age=ages[uid]
#根据用户年龄过滤
#if age>=100 or age<=5:
# continue
correct_status = 0
#根据合法status数量过滤
for status in user['statuses']:
if is_not_good_status(status):
continue
else:
correct_status += 1
if correct_status < 50:
continue
length = []
# text_vector=[numpy.zeros(text_vector_size)]
# time_vector=[numpy.zeros(time_vector_size)]
# image_vector=[numpy.zeros(image_vector_size)]
# for description in user['information']['descriptions']:
# for word in description:
# try:
# image_vector.append(word_vectors[word])
# except:
# continue
# if len(image_vector)==1:
# continue
text = []
for status in user['statuses']:
if is_not_good_status(status):
continue
for word in status['text']:
try:
text.append(word_vectors[word])
except Exception as e:
continue
if len(text) > 100:
continue
#sentence_vector=[]
#for word in status['text']:
# try:
# sentence_vector.append(list(word_vectors[word]))
# except:
# continue
#for i in range(0,len(sentence_vector)-window_size):
# text.append(get_text_convolution(sentence_vector[i:i+window_size]))
# created_at=status['time']
# try:
# hour=get_hour(created_at)
# time_vector.append(time_vectors[hour])
# except Exception as e:
# continue
# pass
# length.append(len(status['text']))
# text_vector_mean=numpy.mean(text_vector,axis=0)
# if len(text)<50:
# continue
text = text[0:100]
text_vector = numpy.array(text) #numpy.max(text_vector,axis=0)
#text_vector=numpy.max(text,axis=0)
text_vector = text_vector.reshape(
(text_vector.shape[0] * text_vector.shape[1]))
# time_vector=numpy.sum(time_vector,axis=0)
#data.append(get_age_class(age))
if user['information']['gender'] == 'm':
all_data_y.append(1)
else:
all_data_y.append(0)
# data=data+list(text_vector_mean)+list(time_vector)+list(numpy.max(image_vector,axis=0))+[numpy.max(length),numpy.min(length),numpy.mean(length),len(length)]
#data=data+list(time_vector)+[numpy.max(length),numpy.min(length),numpy.mean(length),len(length)]
#data=data+list(text_vector)
all_data_x.append(text_vector)
#all_data.append(data)
index += 1
finish_count += 1
bar.cursor.restore()
bar.draw(value=finish_count)
#if finish_count>500:
# break
all_data_x = numpy.array(all_data_x)
all_data_y = numpy.array(all_data_y)
train_set_x = all_data_x[0:index * 3 / 4]
train_set_y = all_data_y[0:index * 3 / 4]
train_set = (train_set_x, train_set_y)
valid_set_x = all_data_x[index * 3 / 4:index * 7 / 8]
valid_set_y = all_data_y[index * 3 / 4:index * 7 / 8]
valid_set = (valid_set_x, valid_set_y)
test_set_x = all_data_x[index * 7 / 8:]
test_set_y = all_data_y[index * 7 / 8:]
test_set = (test_set_x, test_set_y)
pickle.dump((train_set, valid_set, test_set),
open('gender_matrix_with_time.data', 'wb'))
#判断 39 40 41是否都因为drop t_error_maker停止
@dec_progressive
def get_error_status():
while True:
Last_SQL_Error_39 = get_slave_statue(conn39,'show slave status;')['Last_SQL_Error']
Last_SQL_Error_40 = get_slave_statue(conn40,'show slave status;')['Last_SQL_Error']
Last_SQL_Error_41 = get_slave_statue(conn41,'show slave status;')['Last_SQL_Error']
if Last_SQL_Error_39 == Last_SQL_Error_40 == Last_SQL_Error_41 == error_message:
break
else:
time.sleep(1)
if __name__ == '__main__':
MAX_VALUE = 100
bar = Bar(max_value=MAX_VALUE, fallback=True)
bar.cursor.clear_lines(2)
bar.cursor.save()
i=0
#不显示MySQL的warning
filterwarnings('ignore',category=pymysql.Warning)
#连接3306 制造复制异常函数
print(u"连接3306 制造复制异常函数")
error_maker(host='172.16.65.36', port=3306, user='******',password='******',db='fandb',charset='utf8')
conn39 = get_conn('10.0.1.39',3306,'root','mysql')
conn40 = get_conn('10.0.1.40',3306,'root','mysql')
conn41 = get_conn('10.0.1.41',3306,'root','mysql')
#判断 39 40 41是否都因为drop t_error_maker停止
print(u"判断 39 40 41是否都因为drop t_error_maker停止")
def process_request(obj):
# check if year is mentioned
if obj.year:
_year = obj.year
else:
_year = datetime.datetime.now().strftime("%Y")
if obj.month:
_month = obj.month
else:
_month = datetime.datetime.now().strftime("%B")
if obj.day:
_day = obj.day
else:
_day = datetime.datetime.now().strftime("%d")
_id = None
if obj.id:
_id = obj.id
_amount = None
if obj.amount:
_amount = obj.amount
_tag = None
if obj.tag:
_tag = obj.tag
# create a database connection
path = os.path.join(os.path.dirname(__file__,), '..', '..', 'data', _year+'.db')
conn = sqlite3.connect(path)
# create object of Bar
prog = Bar(filled_color=2, title=u'Updating record , please wait ....')
# create cursor object and clear lines
prog.cursor.clear_lines(2)
# save the state of the cursor
prog.cursor.save()
for i in range(101):
time.sleep(0.02)
prog.cursor.restore()
prog.draw(i)
# check if _id is present ,if yes delete that particular id record
if _id:
if _amount:
cursor = conn.execute("UPDATE '%s' SET AMOUNT='%f' WHERE DAY='%d' AND ID='%d'" %(_month, float(_amount),int(_day), int(_id)))
if _tag:
cursor = conn.execute("UPDATE '%s' SET TAG='%s' WHERE DAY='%d' AND ID='%d'" %(_month, _tag,int(_day), int(_id)))
else:
#display records
if _amount:
cursor = conn.execute("UPDATE '%s' SET AMOUNT='%f' WHERE DAY='%d'" %(_month, float(_amount),int(_day)))
if _tag:
cursor = conn.execute("UPDATE '%s' SET TAG='%s' WHERE DAY='%d'" %(_month, _tag ,int(_day)))
conn.commit()
conn.close()
def simul_css(CataSect, _CssImg, cssbands, filtnumb, npi):
# print('Process'+str(npi)
CssHei, CssWid = _CssImg.shape
OutSecStr = ''
LenCatSec = len(CataSect)
procedi = 0
if IfProgBarOn == True:
bar = Bar(max_value=LenCatSec,
empty_color=7,
filled_color=18 + npi * 6,
title='Process-' + str(npi))
bar.cursor.clear_lines(1)
bar.cursor.save()
for procedi, cataline in enumerate(CataSect, 1):
np.random.seed()
ident = str(cataline['IDENT'])
objwind = csstpkg.windcut(_CssImg, cataline)
if objwind is None:
if DebugTF == True:
print('--- Object window cutting error ---')
continue
# DataArr2Fits(objwind, ident+'_convwin.fits')
objwinshape = objwind.shape
objwind.data = objwind.data * ExpCssFrm
WinImgBands = np.zeros(
(len(cssbands), objwinshape[0],
objwinshape[1])) # 3-D array contains images of all the cssbands
if IfPlotObjWin == True:
csstpkg.PlotObjWin(objwind, cataline)
outcatrowi = [ident, cataline['Z_BEST']]
if DebugTF == True:
print(' '.join([
ident, '\nRA DEC:',
str(cataline['RA']),
str(cataline['DEC'])
]))
# Photometry for the central object on the convolved window
ObjWinPhot_DeBkg = csstpkg.CentrlPhot(objwind.data,
id=str(outcatrowi[0]) +
" ConvWdW DeBkg")
ObjWinPhot_DeBkg.Bkg(idb=str(outcatrowi[0]) + " ConvWdW DeBkg",
debug=DebugTF,
thresh=1.5,
minarea=10,
deblend_nthresh=32,
deblend_cont=0.01)
ObjWinPhot_DeBkg.Centract(idt=str(outcatrowi[0]) + " ConvWdW DeBkg",
thresh=2.5,
deblend_nthresh=32,
deblend_cont=0.1,
minarea=10,
debug=DebugTF,
sub_backgrd_bool=True)
if ObjWinPhot_DeBkg.centobj is np.nan:
if DebugTF == True:
print('--- No central object detected in convolved image ---')
continue
else:
ObjWinPhot_DeBkg.KronR(idk=str(outcatrowi[0]) + " ConvWdW",
debug=DebugTF,
mask_bool=True)
NeConv_DeBkg, ErrNeConv_DeBkg = ObjWinPhot_DeBkg.EllPhot(
ObjWinPhot_DeBkg.kronr, mask_bool=True)
if ((NeConv_DeBkg <= 0) or (NeConv_DeBkg is np.nan)):
if DebugTF == True:
print(
'NeConv_DeBkg for a winimg <= 0 or NeConv_DeBkg is np.nan')
continue
noisebkg_conv = ObjWinPhot_DeBkg.bkg.background_rms_median
if DebugTF == True:
print('self.bkg Flux & ErrFlux =',
ObjWinPhot_DeBkg.bkg.background_median,
ObjWinPhot_DeBkg.bkg.background_rms_median)
print('Class processed NeConv_DeBkg & ErrNeConv_DeBkg:',
NeConv_DeBkg, ErrNeConv_DeBkg)
# Read model SED to NDArray
# modsednum = cataline['MOD_BEST']
sedname = seddir + 'Id' + '{:0>9}'.format(ident) + '.spec'
modsed, readflag = csstpkg.readsed(sedname)
if readflag == 1:
modsed[:, 1] = csstpkg.mag2flam(
modsed[:, 1], modsed[:, 0]
) # to convert model SED from magnitude to f_lambda(/A)
else:
print('model sed not found.')
continue
bandi = 0
NeBands = []
magsimorigs = []
scalings = []
for cssband, numb in zip(cssbands, filtnumb):
expcss = 150. * numb # s
# cssbandpath = thrghdir+cssband+'.txt'
NeABand0 = csstpkg.NeObser(modsed, cssband, expcss,
TelArea) # *cataline['SCALE_BEST']
if NeABand0 < 1:
continue
magaband0 = csstpkg.Ne2MagAB(NeABand0, cssband, expcss, TelArea)
delmag = magaband0 - float(cataline['MOD_' + cssband + '_css'])
magsimorig_band = magaband0 - delmag
NeABand = NeABand0 / 10**(-0.4 * delmag)
# print(NeABand0, magaband0, delmag)
NeBands.append(NeABand)
if DebugTF == True:
print(' '.join(
[cssband, 'band model electrons = ',
str(NeABand), 'e-']))
print('MOD_' + cssband + '_css =',
cataline['MOD_' + cssband + '_css'])
magsimorigs.append(
csstpkg.Ne2MagAB(NeABand, cssband, expcss, TelArea))
print('Magsim_' + cssband + ' =', magsimorigs[bandi])
Scl2Sed = NeABand / NeConv_DeBkg
scalings.append(Scl2Sed)
if DebugTF == True:
print(ident, 'Scaling Factor: ', Scl2Sed)
# ZeroLevel = config.getfloat('Hst2Css', 'BZero')
SkyLevel = 0 #csstpkg.backsky[cssband] * expcss
DarkLevel = 0 #config.getfloat('Hst2Css', 'BDark') * expcss
RNCssFrm = config.getfloat('Hst2Css', 'RNCss')
# IdealImg = objwind.data * Scl2Sed + SkyLevel + DarkLevel # e-
IdealImg = ObjWinPhot_DeBkg.data_bkg * Scl2Sed # + SkyLevel + DarkLevel # e-
# IdealImg[IdealImg < 0] = 0
if DebugTF == True:
# csstpkg.DataArr2Fits(IdealImg/Gain, 'Ideal_Zero_Gain_check_'+ident+'_'+cssband+'.fits')
# Testing photometry for the scaled convolved window's central object
ObjWinPhot = csstpkg.CentrlPhot(IdealImg,
id=(ident + " SclTesting"))
try:
ObjWinPhot.Bkg(idb=ident + " SclTesting",
debug=DebugTF,
thresh=1.5,
minarea=10,
deblend_nthresh=32,
deblend_cont=0.01)
except Exception as e:
# print(NeConv_DeBkg, NeABand, IdealImg)
continue
ObjWinPhot.Centract(idt=ident + " SclTesting",
thresh=2.5,
deblend_nthresh=32,
deblend_cont=0.1,
minarea=10,
debug=DebugTF,
sub_backgrd_bool=False)
if ObjWinPhot.centobj is np.nan:
print(
'--- No central object detected in testing photometry image---'
)
continue
else:
ObjWinPhot.KronR(idk=ident + " SclTesting",
debug=DebugTF,
mask_bool=True)
NeConv, ErrNeConv = ObjWinPhot.EllPhot(ObjWinPhot.kronr,
mask_bool=True)
print(' '.join([
'Model electrons:',
str(NeABand), '\nTesting Photometry After scaling:',
str(NeConv)
]))
BkgNoiseTot = (SkyLevel + DarkLevel + RNCssFrm**2 * numb)**0.5
if BkgNoiseTot > noisebkg_conv * Scl2Sed:
Noise2Add = (BkgNoiseTot**2 -
(noisebkg_conv * Scl2Sed)**2)**0.5
else:
Noise2Add = 0
if DebugTF == True:
print('Added Noise ' + cssband + ' band: ', Noise2Add)
# ImgPoiss = np.random.poisson(lam=IdealImg, size=objwinshape)
ImgPoiss = IdealImg
NoisNormImg = csstpkg.NoiseArr(objwinshape,
loc=0,
scale=Noise2Add,
func='normal')
# DigitizeImg = np.round((ImgPoiss + NoisNorm + ZeroLevel) / Gain)
DigitizeImg = (ImgPoiss + NoisNormImg) / Gain
# DigitizeImg = IdealImg/Gain
if DebugTF == True:
csstpkg.DataArr2Fits(
DigitizeImg, 'Ideal_Zero_Gain_RN_check_' + ident + '_' +
cssband + '.fits')
WinImgBands[bandi, ::] = DigitizeImg
bandi = bandi + 1
if DebugTF == True:
print('Stack all bands and detect objects:')
WinImgStack = WinImgBands.sum(0)
# print(WinImgStack.shape)
# AduStack, ErrAduStack, ObjectStack, KronRStack, MaskStack = septract(WinImgStack, id=str(outcatrowi[0])+" Stack", debug=DebugTF, thresh=1.2, minarea=10)
StackPhot = csstpkg.CentrlPhot(WinImgStack, id=ident + " Stack")
StackPhot.Bkg(idb=ident + " Stack",
debug=DebugTF,
thresh=1.5,
minarea=10)
StackPhot.Centract(idt=ident + " Stack",
thresh=1.5,
minarea=10,
deblend_nthresh=32,
deblend_cont=0.1,
debug=DebugTF)
if StackPhot.centobj is np.nan:
if DebugTF == True:
print('No central object on STACK image.')
continue
else:
StackPhot.KronR(idk=ident + " Stack",
debug=DebugTF,
mask_bool=True)
AduStack, ErrAduStack = StackPhot.EllPhot(StackPhot.kronr,
mask_bool=True)
if AduStack is np.nan:
if DebugTF == True:
print('RSS error for STACK image.')
continue
if DebugTF == True:
csstpkg.PlotKronrs(WinImgStack, StackPhot)
bandi = 0
for cssband, numb in zip(cssbands, filtnumb):
expcss = 150. * numb # s
if DebugTF == True:
plt.hist(
WinImgBands[bandi, ::].flatten(),
bins=np.arange(30) - 15,
)
plt.title(' '.join([cssband, 'simul image']))
plt.show()
SameApObj = csstpkg.CentrlPhot(WinImgBands[bandi, ::],
id=ident + ' ' + cssband +
' band CentralExtract')
SameApObj.Bkg(idb=ident + ' ' + cssband + ' band CentralExtract',
debug=DebugTF,
thresh=1.5,
minarea=10)
SameApObj.Centract(idt=ident + ' ' + cssband +
' band CentralExtract',
thresh=1.2,
minarea=10,
deblend_nthresh=32,
deblend_cont=0.1,
debug=DebugTF,
sub_backgrd_bool=False)
if SameApObj.centobj is np.nan:
if DebugTF == True:
print('No central object on simulated image.')
SNR = -99
MagObser = -99
ErrMagObs = -99
# AduObser, ErrAduObs = csstpkg.septractSameAp(WinImgBands[bandi, ::], StackPhot, ObjWinPhot_DeBkg.centobj, ObjWinPhot_DeBkg.kronr, mask_det=StackPhot.mask_other, debug=DebugTF, annot=cssband+'_cssos', thresh=1.2, minarea=10, sub_backgrd_bool=False)
# if AduObser>0:
# MagObser = -2.5 * math.log10(AduObser) + magab_zeros[bandi]
# ErrMagObs = -1
# ErrAduTot = (ErrAduObs ** 2 + (noisebkg_conv * scalings[bandi]) ** 2) ** 0.5
# SNR = AduObser / ErrAduTot
# else:
# SNR = -99
# MagObser = -99
# ErrMagObs = -99
else:
# AduObser, ErrAduObs = csstpkg.septractSameAp(WinImgBands[bandi, ::], StackPhot, ObjWinPhot_DeBkg.centobj, ObjWinPhot_DeBkg.kronr, mask_det=StackPhot.mask_other, debug=DebugTF, annot=cssband+'_cssos', thresh=1.2, minarea=10, sub_backgrd_bool=False)
AduObser, ErrAduObs = csstpkg.septractSameAp(
WinImgBands[bandi, ::],
StackPhot,
StackPhot.centobj,
StackPhot.kronr,
mask_det=StackPhot.mask_other,
debug=DebugTF,
annot=cssband + '_cssos',
thresh=1.2,
minarea=10,
sub_backgrd_bool=False)
if AduObser > 0:
ErrAduTot = (ErrAduObs**2 +
(noisebkg_conv * scalings[bandi])**2)**0.5
SNR = AduObser / ErrAduTot
# MagObser = Ne2MagAB(AduObser*Gain,cssband,expcss,TelArea)
MagObser = -2.5 * math.log10(AduObser) + magab_zeros[bandi]
ErrMagObs = 2.5 * math.log10(1 + 1 / SNR)
if DebugTF == True:
if ((cssband == 'r') &
(np.abs(MagObser -
cataline['MOD_' + cssband + '_css']) > 1)):
csstpkg.DataArr2Fits(objwind.data,
ident + '_convwin_r.fits')
csstpkg.DataArr2Fits(WinImgStack,
ident + '_stack.fits')
else:
SNR = -99
MagObser = -99
ErrMagObs = -99
if DebugTF == True:
npixel = math.pi * (
ObjWinPhot_DeBkg.centobj['a'] * csstpkg.kphotpar *
ObjWinPhot_DeBkg.kronr) * (
ObjWinPhot_DeBkg.centobj['b'] * csstpkg.kphotpar *
ObjWinPhot_DeBkg.kronr)
print(' '.join([
cssband, 'band model e- =',
str(NeBands[bandi]), 'e-'
]))
print(' '.join([
cssband, 'band simul e- =',
str(AduObser * Gain), 'e-', ' ErrNe=',
str(ErrAduTot * Gain)
]))
# print(AduObser, Gain, NeBands[bandi], -2.5*math.log10(AduObser*Gain/NeBands[bandi]))
print('SNR =', AduObser / ErrAduTot)
print('Npixel =', npixel)
print(' '.join([
cssband, 'band mag_model = ',
str(cataline['MOD_' + cssband + '_css']), '(AB mag)'
]))
print(' '.join([
cssband, 'band Magsim_orig = ',
str(magsimorigs[bandi]), '(AB mag)'
]))
print(' '.join([
cssband, 'band Mag_simul = ',
str(MagObser), '(AB mag)'
]))
print(' '.join([
cssband, 'band magerr_simul = ',
str(ErrMagObs), '(AB mag)'
]))
print(' '.join([
'Magsim - Magsimorig =',
str(MagObser - magsimorigs[bandi])
]))
outcatrowi = outcatrowi + [
cataline['MOD_' + cssband + '_css'], MagObser, ErrMagObs, SNR
]
bandi = bandi + 1
del WinImgBands
colnumb = len(outcatrowi)
OutRowStr = ('{} ' +
(colnumb - 1) * '{:8.3f}').format(*outcatrowi) + '\n'
OutSecStr = OutSecStr + OutRowStr
if IfProgBarOn == True:
bar.cursor.restore() # Return cursor to start
bar.draw(value=procedi)
if IfProgBarOn == True:
bar.cursor.restore() # Return cursor to start
bar.draw(value=bar.max_value) # Draw the bar!
# OutCatSecQueue.put(OutSecStr)
# _FinishQueue.put(1)
# write_lock.acquire()
with write_lock:
OutCssCat.write(OutSecStr)
OutCssCat.flush()
# write_lock.release()
print('\n')
def simul_css(CataSect, cssbands, filtnumb, npi):
# print('Process'+str(npi)
OutSecStr = ''
LenCatSec = len(CataSect)
procedi = 0
if IfProgBarOn == True:
bar = Bar(max_value=LenCatSec,
empty_color=7,
filled_color=18 + npi * 6,
title='Process-' + str(npi))
bar.cursor.clear_lines(1)
bar.cursor.save()
for procedi, cataline in enumerate(CataSect, 1):
np.random.seed()
ident = str(cataline['IDENT'])
outcatrowi = [ident, cataline['Z_BEST']]
if DebugTF == True:
print(' '.join([
ident, '\nRA DEC:',
str(cataline['RA']),
str(cataline['DEC'])
]))
sedname = seddir + 'Id' + '{:0>9}'.format(ident) + '.spec'
modsed = csstpkg.readsed(sedname)
modsed[:, 1] = csstpkg.mag2flam(
modsed[:, 1],
modsed[:,
0]) # to convert model SED from magnitude to f_lambda(/A)
for bandi, cssband in enumerate(cssbands):
mag_en = csstpkg.mag_ener(modsed, cssband, magab_ener_zeros[bandi])
mag_ph = csstpkg.mag_phot(modsed, cssband, magab_phot_zeros[bandi])
if DebugTF == True:
# print(' '.join([cssband, 'band model electrons = ', str(NeABand), 'e-']))
print('MOD_' + cssband + '_css =',
cataline['MOD_' + cssband + '_css'])
print('Magsim_' + cssband + '_photon =', mag_ph)
print('Magsim_' + cssband + '_energy =', mag_en)
outcatrowi = outcatrowi + [
cataline['MOD_' + cssband + '_css'], mag_en, mag_ph
]
# bandi = bandi + 1
# del WinImgBands
colnumb = len(outcatrowi)
OutRowStr = ('{} ' +
(colnumb - 1) * '{:8.3f}').format(*outcatrowi) + '\n'
OutSecStr = OutSecStr + OutRowStr
if IfProgBarOn == True:
bar.cursor.restore() # Return cursor to start
bar.draw(value=procedi)
if IfProgBarOn == True:
bar.cursor.restore() # Return cursor to start
bar.draw(value=bar.max_value) # Draw the bar!
with write_lock:
OutCssCat.write(OutSecStr)
OutCssCat.flush()
# write_lock.release()
print('\n')
'''convert img to hdf5
'''
import glob
import pandas as pd
from keras.preprocessing import image
import numpy as np
from progressive.bar import Bar
imglist = glob.glob('data/train-jpg/*.jpg')
allimg = sorted(imglist, key=lambda x: int(x[21:].split('.')[0]))
imgsize = 224
totalimg = len(imglist)
alldata = np.zeros([totalimg, imgsize * imgsize * 3], dtype=np.uint8)
pb = Bar(max_value=totalimg, fallback=True)
pb.cursor.clear_lines(2)
pb.cursor.save()
for index, img in enumerate(allimg):
img = image.load_img(img, target_size=(224, 224))
alldata[index, :] = image.img_to_array(img).reshape(-1)
pb.cursor.restore()
pb.draw(index)
pddata = pd.DataFrame(alldata)
pddata.to_hdf('data/train_amazon.hdf5', 'train_amazon')
data.batch_size_test(1000)
lr = 1e-3
gamma = 1
beta_1 = 0.9
beta_2 = 0.999
total_epoch = 100
loss_cache = 10
for epoch in xrange(1, total_epoch + 1):
print 'Epoch: {}/{}'.format(epoch, total_epoch)
# Training (Mini-batch)
now = time.time()
data.shuffle()
bar = Bar(max_value=n)
bar.cursor.clear_lines(2) # Make some room
bar.cursor.save() # Mark starting line
for _ in xrange(data.batch_run()):
net.input(data.next_batch())
net.forward()
net.backward(lr, beta_1, beta_2, epoch)
bar.cursor.restore() # Return cursor to start
bar.draw(value=data.get_count())
t = time.time() - now
acc, loss = net.get_record()
loss_avg = np.array(loss).mean()
loss_diff = loss_avg - loss_cache
loss_cache = loss_avg
print 'Acc: ', np.array(acc).mean()
print 'Loss: ', loss_avg