def run(self):
global threads
docs = get_docs(CORPUS_PATH)
mapper = Mapper()
i = 1
for files in self.__chunk(docs):
doc_contents = []
for f in files:
with open(f, 'r') as d:
doc_contents.append(d.read())
doc_contents = ''.join(doc_contents)
self.__run_batch(parse(doc_contents), i, mapper)
i += 1
print "Writing the mapper to file -------------------------------------"
mapper.write(self.name)
print "Writing DocLengths to file --------------------------------------"
self.__writeDocLengths()
while len(self.catalogs) != 1:
print self.chunk_catalog()
for pair in self.chunk_catalog():
print pair
if len(pair) != 2:
break
else:
cat1 = self.catalogs[pair[0]]
cat2 = self.catalogs[pair[1]]
self.__merge(cat1, cat2, pair)
print "Writing the catalog to file for later use -----------------------"
Catalog.write(self.catalogs, self.name)
def __dirSearch1(self, _rootCatalog, _ndeep=10):
"""
文件夹遍历,结果存放在自定义的目录对象(CatalogList)，用于存放一个文件夹包含的所有文件(CatalogItem)对象
:param _rootCatalog: 目录对象(CatalogList)
:param _ndeep: 允许的最大递归索引深度
:return:
"""
if isinstance(_rootCatalog, Catalog.CatalogList):
dirPath = _rootCatalog.dirPath()
dirlist = []
if os.path.isdir(dirPath) and (dirPath != '/'):
dirlist = os.listdir(dirPath)
if len(dirlist) <= 0:
return False
dirHaveRightDoc = False
for t_file in dirlist:
# print os.path.join(dirPath, t_file)
if not os.path.isdir(os.path.join(dirPath, t_file)):
if self.isFIleInFilter(t_file):
_rootCatalog.addItem(Catalog.CatalogItem(t_file))
dirHaveRightDoc = dirHaveRightDoc or True
else:
t_childrenlist = Catalog.CatalogList(os.path.join(dirPath, t_file))
t_catalogItem = Catalog.CatalogItem(t_file, _children=t_childrenlist)
_rootCatalog.addItem(t_catalogItem)
if _ndeep >= 0:
a = self.__dirSearch1(t_childrenlist, _ndeep=_ndeep-1)
dirHaveRightDoc = dirHaveRightDoc or a
return dirHaveRightDoc
def insert(table_name: str, values: list):
time_start = time.time()
Catalog.not_exists_table(table_name)
Catalog.check_types_of_table(table_name, values)
linenum = Buffer.insert_record(table_name, values)
Index.insert_into_table(table_name, values, linenum)
time_end = time.time()
print(" time elapsed : %fs." % (time_end - time_start))
def delete(table_name: str, where: list = None):
time_start = time.time()
Catalog.not_exists_table(table_name)
Catalog.check_select_statement(table_name, ['*'], where) # 从insert中借用的方法
col = Catalog.get_column_dic(table_name)
pklist = Buffer.delete_record(table_name, col, where)
Index.delete_from_table(table_name, pklist)
time_end = time.time()
print(" time elapsed : %fs." % (time_end - time_start))
def create_table(table_name: str, attributes: list, pk: str):
time_start = time.time()
Catalog.exists_table(table_name)
Index.create_table(table_name)
Catalog.create_table(table_name, attributes, pk)
Buffer.create_table(table_name)
time_end = time.time()
print("Successfully create table '%s', time elapsed : %fs." %
(table_name, time_end - time_start))
def drop_table(table_name: str):
time_start = time.time()
Catalog.not_exists_table(table_name)
Catalog.drop_table(table_name)
Buffer.drop_table(table_name)
Index.delete_table(table_name)
time_end = time.time()
print("Successfully drop table '%s', time elapsed : %fs." %
(table_name, time_end - time_start))
def test_old_catalog_init(tmpdir, old_catalog_file):
c = Catalog.from_old_catalog(old_catalog_file, catalog_path=tmpdir)
# Verify the catalog is nonempty and contains the expected data
assert len(c) == 4
for dsname in ["wine_reviews_130k", "wine_reviews_150k", 'wine_reviews_130k_varietals_75', 'wine_reviews']:
assert dsname in c
# Should fail, as it already exists
with pytest.raises(FileExistsError):
c = Catalog.from_old_catalog(old_catalog_file, catalog_path=tmpdir)
# Should succeed, as replace is set
c = Catalog.from_old_catalog(old_catalog_file, catalog_path=tmpdir, replace=True)
def catalog(tmpdir):
"""Create a test catalog"""
# Setup
# tmpdir should be empty when we get here
c = Catalog.create(catalog_path=tmpdir)
yield c
def receiveCommand(self): while not self._interruptFlag: # Waiting for any two lines (so that the VideoClient only gets one catalog per request) self._commandSocket.nextLine() self._commandSocket.nextLine() self._commandSocket.send(Catalog.asHttp()) self.kill()
def select(table_name: str, attributes: list, where: list = None):
time_start = time.time()
Catalog.not_exists_table(table_name)
Catalog.check_select_statement(table_name, attributes, where)
#Index.select_from_table(table_name, attributes, where)
col_dic = Catalog.get_column_dic(table_name)
results = Buffer.find_record(table_name, col_dic, where)
numlist = []
if attributes == ['*']:
attributes = list(col_dic.keys())
numlist = list(col_dic.values())
else:
for att in attributes:
numlist.append(col_dic[att])
print_select(attributes, numlist, results)
time_end = time.time()
print(" time elapsed : %fs." % (time_end - time_start))
def GetCatalogTree(self, dirPath):
"""
使用__dirSearch1()方法搜索目录,产生CatalogList对象,存放独有成员self.catalogTree
:param dirPath:
:return:
"""
self.catalogTree = Catalog.CatalogList(dirPath)
self.__dirSearch1(self.catalogTree)
return self.catalogTree
def analysis( args ):
from Timer import Timer
import Catalog
with Timer('analysis') as t:
if args['image_fits_input'] == 'none':
with Timer('events generated') as t:
events = Simulation.simulate_events( args )
if args['image_fits_output'] != 'none':
with Timer('fits image saved at ' + args['image_fits_output'] ) as t:
events.generate_image( events, args['image_fits_output'] )
args['image_fits_input'] = args['image_fits_output']
image = Image.read_image_from_file( args['image_fits_input'] )
with Timer('hits extracted') as t:
hits = image.extract_hits( mode = 'cluster', threshold = args['extraction_threshold'], border = args['extraction_border'] )
n = args['extraction_border']+1
hits = hits.add_number_of_pixel_levels( range(n) )
hits = hits.add_energy_levels( range(n) )
hits = hits.add_barycenter_levels( range(n) )
hits = hits.add_variance_levels( range(n) )
print( 'hits', hits.dtype.names )
if args['image_fits_input'] == 'none':
hits = hits.match_simulated_events( events, lvl=1, length=3 )
if args['catalog_root_output'] != 'none':
with Timer('root catalog at ' + args['catalog_root_output'] ) as t:
Catalog.build_new_catalog_from_recarray( hits, args['catalog_root_output'], None )
#with Timer('size like') as t:
#hits.add_sizelike( 3, args['charge_gain'], args['readout_noise'], fast = False, tol = 1e-3 )
#t( len(hits) )
#with Timer('write catalog sizelike') as t:
#Catalog.build_new_catalog_from_recarray( hits, 'catalog_test.root', None )
if args['reconstruction_image'] != 'none':
with Timer('plot reconstruction') as t:
show_reconstruction( args['reconstruction_image'], events, image, hits, zoom=[[0,200],[0,200]] )
def SetCatalogObj(self, dirPath, _ndeep=10):
"""
使用__dirSearch()方法搜索目录,并返回结果
:param dirPath:
:param _ndeep:索引深度
:return:self.__CatalogObj:目录对象CatalogList
"""
self.__CatalogObj = Catalog.CatalogList(dirPath)
self.__dirSearch(self.__CatalogObj, _ndeep)
return self.GetCatalogObj()
def mainProgram():
print("- Welcome to version 0.6 of Filofax")
print(
"- Type the first letter of the alternative in the menu to execute it."
)
try:
filofax = Catalog(FILENAME)
except UnboundLocalError:
print('File ' + FILENAME + ' not found!')
printCurrent(filofax)
printMenu()
choice = choose()
while choice != 'Q':
if choice == 'N':
filofax.changeDate(1)
if choice == 'P':
filofax.changeDate(-1)
if choice == 'D':
addNote(filofax)
if choice == 'E':
removeNote(filofax)
if choice == 'A':
addPage(filofax)
if choice == 'R':
removePage(filofax)
if choice == 'S':
showPages(filofax)
if choice == 'M':
showMonth(filofax)
if choice == 'J':
date = chooseDate()
filofax.setDate(date)
if choice == 'C':
date = chooseDate()
showPage(filofax, date)
printCurrent(filofax)
printMenu()
choice = choose()
filofax.save(FILENAME)
def mainProgram():
print("- Welcome to version 0.6 of Filofax")
print("- Type the first letter of the alternative in the menu to execute it.")
try:
filofax = Catalog(FILENAME)
except UnboundLocalError:
print("File " + FILENAME + " not found!")
printCurrent(filofax)
printMenu()
choice = choose()
while choice != "Q":
if choice == "N":
filofax.changeDate(1)
if choice == "P":
filofax.changeDate(-1)
if choice == "D":
addNote(filofax)
if choice == "E":
removeNote(filofax)
if choice == "A":
addPage(filofax)
if choice == "R":
removePage(filofax)
if choice == "S":
showPages(filofax)
if choice == "M":
showMonth(filofax)
if choice == "J":
date = chooseDate()
filofax.setDate(date)
if choice == "C":
date = chooseDate()
showPage(filofax, date)
printCurrent(filofax)
printMenu()
choice = choose()
filofax.save(FILENAME)
def create_index(index_name: str, table_name: str, indexed_attr: str):
Catalog.exists_index(index_name)
Catalog.create_index(index_name, table_name, indexed_attr)
Index.create_index(index_name, table_name, indexed_attr)
# -*-coding:Utf-8 -*
# Y:\3IF\RE\TP-1\src
import Catalog
from handlers import HandlerFactory
from handlers.MultiCastCatalogHandler import *
from sockets.TcpSocket import *
from sockets.SocketManager import *
(serverAddress, catalogPort) = Catalog.parse('catalog/startup.txt')
Catalog.addMediaToResourceManager()
connectionProperties = Catalog.getConnectionProperties()
HandlerFactory.setConnectionProperties(connectionProperties)
# Instanciate one SocketManager per media + one for the catalog
# (these will accept all new control connections)
servers = []
for properties in connectionProperties:
server = SocketManager(serverAddress, properties['port'], properties['protocol'])
server.start()
servers.append(server)
# Instanciate one more to serve the catalog via multicast
multiCastCatalogHandler = MultiCastCatalogHandler('225.6.7.8', 4567)
multiCastCatalogHandler.start()
input("Press enter to shutdown server...\r\n")
# Once any input was given, we start closing down the connections
# The script will end when all the connections are released
self.catalog.remove_inactive_ws()
# remove inactive fridges
self.catalog.remove_inactive_fridge()
print("Inactive fridges and web services removed.")
time.sleep(5*60)
########################################## MAIN FUNCTION ############################################
if __name__ == '__main__':
conf = {
'/': {
'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
'tools.sessions.on': True,
}
}
catalog = Catalog("Catalog.json")
inactiveThread = RemoveInactiveThread(catalog)
inactiveThread.start()
cherrypy.tree.mount(Catalog_REST(catalog), '/', conf)
cherrypy.config.update({'server.socket_host': '0.0.0.0'})
cherrypy.config.update({'server.socket_port': 8080})
cherrypy.engine.start()
cherrypy.engine.block()
def setup_method(self, method):
print "Entered setup"
print method
self.location = "location"
self.webserver = "webserver"
self.dut = Catalog(self.location, self.webserver)
def Icol_corr(cat,
catname,
catIDs,
RAo,
DECo,
radius,
insMags,
insMagerrs,
catMags,
catMagerrs,
plot=True):
#essential extra imports
from scipy.optimize import curve_fit
from SNAP.Analysis.LCFitting import linfunc
import Catalog as ctlg
#fetch V band magnitudes
if cat == 'phot':
IDV, RAV, DECV, catMV, catMerrV = ctlg.catPhot(catname, band='V')
elif cat == 'dprs':
IDV, RAV, DECV, catMV, catMerrV = ctlg.catDPRS(catname, band='V')
elif cat == 'diff':
IDV, RAV, DECV, catMV, catMerrV = ctlg.catDiff(catname, band='V')
elif cat == 'aavso':
fovam = 2.0 * radius * 0.4 / 60.0 #arcmin radius in KMT scaling
IDV, RAV, DECV, catMV, catMerrV = ctlg.catAAVSO(RAo[0],
DECo[0],
fovam,
'V',
out=catname)
#compute V-I
I, Ierr = [], []
KI, KIerr = [], []
V, Verr = [], []
for i in range(len(catMV)):
if IDV[i] in catIDs:
Iid = list(catIDs).index(IDV[i])
I.append(catMags[Iid])
Ierr.append(catMagerrs[Iid])
KI.append(insMags[Iid])
KIerr.append(insMagerrs[Iid])
V.append(catMV[i])
Verr.append(catMerrV[i])
I, Ierr = np.array(I), np.array(Ierr)
KI, KIerr = np.array(KI), np.array(KIerr)
V, Verr = np.array(V), np.array(Verr)
VI = V - I
VI_err = np.sqrt(np.square(Verr) + np.square(Ierr))
#photometric solution color dependence
dI = I - KI
dI_err = np.sqrt(Ierr**2 + KIerr**2)
#average B-V color
VI_err = [VI_err[i] if VI_err[i] > 0 else 0.0005 for i in range(len(VI))]
w = 1 / np.square(VI_err)
VI_mean = np.sum(VI * w) / np.sum(w)
VI_merr = np.sqrt(1 / np.sum(w))
print "Average color (V-I):", VI_mean, "+/-", VI_merr
#make color correlation plot
if plot:
#essential additional import
import matplotlib.pyplot as plt
#fit color dependence
plt.title("I band dependence on V-I")
plt.errorbar(VI, dI, xerr=VI_err, yerr=dI_err, fmt='k+', zorder=1)
popt, pcov = curve_fit(linfunc,
VI,
dI,
p0=[0.27, 27.8],
sigma=dI_err,
absolute_sigma=True)
perr = np.sqrt(np.diag(pcov))
colsol = linfunc(VI, *popt)
#mask out 3sig deviators
mask = np.absolute(dI - colsol) < 3 * np.std(dI - colsol)
plt.scatter(VI[mask], dI[mask], c='r')
popt, pcov = curve_fit(linfunc,
VI[mask],
dI[mask],
p0=[0.27, 27.8],
sigma=dI_err[mask],
absolute_sigma=True)
perr = np.sqrt(np.diag(pcov))
colsol = linfunc(VI[mask], *popt)
print "Color correlation:", popt, perr
print "Nstar:", len(VI[mask])
print "Pearson:", np.corrcoef(VI[mask], dI[mask])
plt.plot(VI[mask], colsol, zorder=2)
plt.ylabel("i - inst")
plt.xlabel("V - i")
plt.show()
#return mean color of reference stars
return VI_mean, VI_merr
def sendCatalog(self): self._isTimerRunning = False self._dataSocket.send(Catalog.asText()) self.startTimer()
def show_table(table_name: str):
Catalog.not_exists_table(table_name)
Catalog.show_table(table_name) # 显示属性名、类型、大小（char）、是否unique
def save():
Catalog.__finalize__()
Index.__finalize__()
Buffer.__finalize__()
print("All tables have been saved.")
class REST_Catalog(object):
exposed=True
def __init__(self):
self.catalog = Catalog("Catalog.json", "Buyers.json", "configCatalog.json", "devices.json")
def GET(self, *uri, **params):
if (len(uri) == 2 and uri[0] == 'get' and uri[1] == 'users'):
return self.catalog.getUsers()
elif (len(uri) == 2 and uri[0] == 'get' and uri[1] == 'userslist'):
return self.catalog.getUsersList()
elif (len(uri) == 2 and uri[0] == 'get' and uri[1] == 'broker'):
return self.catalog.getBroker()
elif (len(uri) == 2 and uri[0] == 'get' and uri[1] == 'devices'):
return self.catalog.getDevices()
def POST(self, *uri):
if len(uri) == 0:
raise cherrypy.HTTPError(400)
mybody = cherrypy.request.body.read()
try:
data = json.loads(mybody)
except:
raise cherrypy.HTTPError(400)
#%% Register device
if (len(uri) == 2 and uri[0] == 'add' and uri[1] == 'device'):
return self.catalog.addDevice(data["NAME"], data["IP"], data["PORT"])
#%% Get user
if (len(uri) == 2 and uri[0] == 'get' and uri[1] == 'user'):
return self.catalog.getUser(data["ID"])
#%% Add user
if (len(uri) == 2 and uri[0] == 'add' and uri[1] == 'user'):
out = self.catalog.addUser(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
#%% Set nickname
elif (len(uri) == 2 and uri[0] == 'set' and uri[1] == 'nickname'):
out = self.catalog.setNickname(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
#%% verify buyer
elif (len(uri) == 2 and uri[0] == 'verify' and uri[1] == 'buyer'):
out = self.catalog.verifyBuyer(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
#%% verify buyer
elif (len(uri) == 2 and uri[0] == 'verify' and uri[1] == 'pw'):
out = self.catalog.verifyPassword(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
#%% set flag
elif (len(uri) == 2 and uri[0] == 'set' and uri[1] == 'flag'):
out = self.catalog.setFlag(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
#%% remove user
elif (len(uri) == 2 and uri[0] == 'remove' and uri[1] == 'user'):
out = self.catalog.removeUser(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
#%% set SMS
elif (len(uri) == 2 and uri[0] == 'set' and uri[1] == 'sms'):
out = self.catalog.setSMS(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
#%% set number
elif (len(uri) == 2 and uri[0] == 'set' and uri[1] == 'number'):
out = self.catalog.setNumber(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
#%% set new mail
elif (len(uri) == 2 and uri[0] == 'set' and uri[1] == 'mail'):
out = self.catalog.setMail(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
#%% set new pw
elif (len(uri) == 2 and uri[0] == 'set' and uri[1] == 'pw'):
out = self.catalog.setPw(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
elif (len(uri) == 2 and uri[0] == 'remove' and uri[1] == 'device'):
out = self.catalog.removeDevice(data)
if out == "Error":
raise cherrypy.HTTPError(400)
else:
return out
def render_mainColumn(self, context):
return [
Counters(self.target),
Catalog.CatalogSection(self.target),
RecentMessages(self.target),
]
t_file, _children=t_childrenlist)
_rootCatalog.addItem(t_catalogItem)
a = self.__dirSearch(t_childrenlist, _ndeep=_ndeep - 1)
dirHaveRightDoc = dirHaveRightDoc or a
return dirHaveRightDoc
def SetCatalogObj(self, dirPath, _ndeep=10):
"""
使用__dirSearch()方法搜索目录,并返回结果
:param dirPath:
:param _ndeep:索引深度
:return:self.__CatalogObj:目录对象CatalogList
"""
self.__CatalogObj = Catalog.CatalogList(dirPath)
self.__dirSearch(self.__CatalogObj, _ndeep)
return self.GetCatalogObj()
def GetCatalogObj(self):
return self.__CatalogObj
if __name__ == '__main__':
dirPath = '/home/yangzheng/myPrograms/MLTools_PyQt4'
test = DocManagerTree()
test.SetCatalogObj(dirPath, 1)
ct = test.GetCatalogObj()
try:
print Catalog.getTree(ct)
except DocException, e:
e.what()
from Catalog import *
from Dialog import Dialog
from Controller import Controller
from View import View
question = Question(
1, 'Привет',
[Answer(1, 'Привет', 2), Answer(2, 'Пока', 0)])
question2 = Question(
2, 'Как дела?',
[Answer(1, 'Хорошо', 3), Answer(3, 'Нормально', 3)])
question3 = Question(3, 'Пока', [Answer(1, 'Пока', 0)])
catalog = Catalog([question, question2, question3])
dialog = Dialog(catalog)
view = View()
controller = Controller(dialog, view)
view.set_controller(controller)
controller.start()
def Bcol_corr(cat,
catname,
catIDs,
RAo,
DECo,
radius,
insMags,
insMagerrs,
catMags,
catMagerrs,
plot=True):
#essential extra imports
from scipy.optimize import curve_fit
from SNAP.Analysis.LCFitting import linfunc
import Catalog as ctlg
#load V band data
if cat == 'aavso':
fovam = 2.0 * radius * 0.4 / 60.0 #arcmin radius in KMT scaling
IDBV, RABV, DECBV, catBV, catBVerr = ctlg.catAAVSO(RAo[0],
DECo[0],
fovam,
'B-V',
out=catname)
B, Berr = [], []
KB, KBerr = [], []
BV, BV_err = [], []
for i in range(len(catBV)):
if IDBV[i] in catIDs:
Bid = list(catIDs).index(IDBV[i])
B.append(catMags[Bid])
Berr.append(catMagerrs[Bid])
KB.append(insMags[Bid])
KBerr.append(insMagerrs[Bid])
BV.append(catBV[i])
BV_err.append(catBVerr[i])
B, Berr = np.array(B), np.array(Berr)
KB, KBerr = np.array(KB), np.array(KBerr)
BV, BV_err = np.array(BV), np.array(BV_err)
else:
#fetch V band magnitudes
if cat == 'phot':
IDV, RAV, DECV, catMV, catMerrV = ctlg.catPhot(catname, band='V')
elif cat == 'dprs':
IDV, RAV, DECV, catMV, catMerrV = ctlg.catDPRS(catname, band='V')
elif cat == 'diff':
IDV, RAV, DECV, catMV, catMerrV = ctlg.catDiff(catname, band='V')
#compute B-V
B, Berr = [], []
KB, KBerr = [], []
V, Verr = [], []
for i in range(len(catMV)):
if IDV[i] in catIDs:
Bid = list(catIDs).index(IDV[i])
B.append(catMags[Bid])
Berr.append(catMagerrs[Bid])
KB.append(insMags[Bid])
KBerr.append(insMagerrs[Bid])
V.append(catMV[i])
Verr.append(catMerrV[i])
B, Berr = np.array(B), np.array(Berr)
KB, KBerr = np.array(KB), np.array(KBerr)
V, Verr = np.array(V), np.array(Verr)
BV = B - V
BV_err = np.sqrt(np.square(Berr) + np.square(Verr))
#photometric solution color dependence
dI = B - KB
dI_err = np.sqrt(Berr**2 + KBerr**2)
#average B-V color
BV_err = [BV_err[i] if BV_err[i] > 0 else 0.0005 for i in range(len(BV))]
w = 1 / np.square(BV_err)
BV_mean = np.sum(BV * w) / np.sum(w)
BV_merr = np.sqrt(1 / np.sum(w))
print "Average color (B-V):", BV_mean, "+/-", BV_merr
#make color correlation plot
if plot:
#essential additional import
import matplotlib.pyplot as plt
#fit color dependence
plt.title("B band dependence on B-V")
plt.errorbar(BV, dI, xerr=BV_err, yerr=dI_err, fmt='k+', zorder=1)
popt, pcov = curve_fit(linfunc,
BV,
dI,
p0=[0.27, 27.8],
sigma=dI_err,
absolute_sigma=True)
perr = np.sqrt(np.diag(pcov))
colsol = linfunc(BV, *popt)
#mask out 3sig deviators
mask = np.absolute(dI - colsol) < 3 * np.std(dI - colsol)
plt.scatter(BV[mask], dI[mask], c='r')
popt, pcov = curve_fit(linfunc,
BV[mask],
dI[mask],
p0=[0.27, 27.8],
sigma=dI_err[mask],
absolute_sigma=True)
perr = np.sqrt(np.diag(pcov))
colsol = linfunc(BV[mask], *popt)
print "Color correlation:", popt, perr
print "Nstar:", len(BV[mask])
print "Pearson:", np.corrcoef(BV[mask], dI[mask])
plt.plot(BV[mask], colsol, zorder=2)
plt.ylabel("B - inst")
plt.xlabel("B - V")
plt.show()
#return mean color of reference stars
return BV_mean, BV_merr
def drop_index(index_name: str):
Catalog.not_exists_index(index_name)
Catalog.drop_index(index_name)
def ClickMag(image,
wcs,
cat,
catname,
radius=500,
band='V',
fwhm=5.0,
limsnr=3.0,
satmag=14.0,
refmag=19.0,
satpix=40000.0,
verbosity=0,
diagnosis=False):
"""
#####################################################################
# Desc: Compute magnitude of objects in image using ref catalog. #
# Select objects by clicking on them. #
# ----------------------------------------------------------------- #
# Imports: #
# ----------------------------------------------------------------- #
# Input #
# ----------------------------------------------------------------- #
# image: numpy array containing image data on which to measure #
# source and reference star photometry. #
# wcs: astropy wcs object, world coordinate system on image. #
# cat: str catalog type (phot, dprs, or diff from Catalog.py) #
# catname: str catalog name. #
# radius; float radius around object in which to take ref stars. #
# band; char observational filter of data. #
# fwhm; float estimate of FWHM on image. #
# limsnr; float signal to noise ratio defining detection limit, #
# if 0.0, then no detection limits are calculated. #
# satmag; float magnitude below which reference stars are #
# considered to be saturated and hence not used. #
# refmag; float magnitude above which reference stars are #
# considered to be reliable, and therefore used. #
# verbosity; int counts verbosity level. #
# ----------------------------------------------------------------- #
# Output #
# ----------------------------------------------------------------- #
# RAo, DECo: float measured equatorial coordinates of sources. #
# Io, SNo: float measured intensities and SNRs of sources. #
# mo, mo_err: float calibrated magnitudes and errors of sources. #
# mlim; float detection limits at source positions. #
#####################################################################
"""
#essential imports
import matplotlib.pyplot as plt
import numpy as np
#essential functions
import Catalog as ctlg
import PSFlib as plib
import Photometry as pht
from MagCalc import PSFError, magnitude
from Analysis.Cosmology import bands, flux_0
#missing
#(RAo,DECo) CHECK
#aperture=None
#psf='1'
#name='object' CHECK
#fitsky=True
#steps
#load all reference stars on image DONE
#plot image DONE
#plot reference stars in blue DONE
#centroid clicked sources -> replot DONE
#after each click, ask for photometry method
#evaluate ra and dec coordinates DONE
#evaluate magnitudes of each using MagCalc DONE
#load photometric reference stars catalog
if verbosity > 0:
print "loading catalog"
if cat == 'phot':
ID, RA, DEC, catM, catMerr = ctlg.catPhot(catname, band=band)
elif cat == 'dprs':
ID, RA, DEC, catM, catMerr = ctlg.catDPRS(catname, band=band)
elif cat == 'diff':
ID, RA, DEC, catM, catMerr = ctlg.catDiff(catname, band=band)
elif cat == 'aavso':
fovam = 4.0 * radius * 0.4 / 60.0 #arcmin radius in KMT scaling
RAo, DECo = [wcs.wcs.crval[0]], [wcs.wcs.crval[1]]
if band == 'I':
if verbosity > 0:
print "Performing AAVSO i -> I band conversion (Jodri 2006)"
IDi, RAi, DECi, catMi, catMierr = ctlg.catAAVSO(RAo[0],
DECo[0],
fovam,
'i',
out=catname)
IDr, RAr, DECr, catMr, catMrerr = ctlg.catAAVSO(RAo[0],
DECo[0],
fovam,
'r',
out=catname)
ID, RA, DEC, catM, catMerr = [], [], [], [], []
for i in range(len(IDi)):
#for each ID in i band
if IDi[i] in IDr:
#if also in r band
j = list(IDr).index(IDi[i]) #here it is
#get I band from i and r
ID.append(IDi[i])
RA.append(RAi[i])
DEC.append(DECi[i])
#Jodri 2006 general stars transform
catMI = 1.083 * catMi[i] - 0.083 * catMr[j] - 0.376
catMIerr = np.sqrt(((catMr[j] - catMi[i]) * 0.006)**2 +
(0.004)**2 + (1.083 * catMierr[i])**2 +
(0.083 * catMrerr[j])**2)
catM.append(catMI)
catMerr.append(catMIerr)
ID, RA, DEC, catM, catMerr = np.array(ID), np.array(RA), np.array(
DEC), np.array(catM), np.array(catMerr)
ID, RA, DEC, catM, catMerr = ctlg.catAAVSO(RAo[0],
DECo[0],
fovam,
'i',
out=catname)
else:
ID, RA, DEC, catM, catMerr = ctlg.catAAVSO(RAo[0],
DECo[0],
fovam,
band,
out=catname)
#convert position of catalog stars to pixels
catX, catY = wcs.all_world2pix(RA, DEC, 0)
catX, catY = catX.astype(float), catY.astype(float)
print max(catM)
#select catalog stars within edges
index = np.logical_and(catX > 80, image.shape[1] - catX > 80)
index = np.logical_and(
index, np.logical_and(catY > 80, image.shape[0] - catY > 80))
#select unsaturated catalog stars
index = np.logical_and(index, catM > satmag)
#select bright enough catalog stars
index = np.logical_and(index, catM < refmag)
#crop values to mask
ID, catX, catY, catRA, catDEC, catM, catMerr = ID[index], catX[
index], catY[index], RA[index], DEC[index], catM[index], catMerr[index]
if len(ID) == 0:
raise PSFError('No reference stars in image.')
if verbosity > 0:
#output selected catalog stars
print "Showing catalog star IDs:"
for i in range(len(ID)):
print ID[int(i)], catX[int(i)], catY[int(i)]
print catRA[int(i)], catDEC[int(i)], catM[int(i)], catMerr[int(i)]
#number of selected catalog stars
Ncat = len(ID)
print Ncat
#plot image of field
fig = plt.figure()
plt.imshow(image, cmap='Greys', vmax=0.0001 * np.amax(image), vmin=0)
plt.scatter(catX, catY, c='b', marker='+')
plt.tight_layout()
x_cens, y_cens = [], []
psfs = []
apers = []
fit_skys = []
#function: record click position on matplotlib image
def onclick(event):
#output click position
if verbosity > 0:
print 'Clicked pixel: x=%d, y=%d' % (event.xdata, event.ydata)
#record clicked position
x_click, y_click = event.xdata, event.ydata
#obtain an aperture around clicked star
intens, x, y = pht.ap_get(image, x_click, y_click, 0, 1 * fwhm)
#computer centroid of clicked star
x_cen = np.sum(intens * x) / intens.sum()
y_cen = np.sum(intens * y) / intens.sum()
print 'Centroid pixel: x=%d, y=%d' % (x_cen, y_cen)
#Should we keep the star
keep = raw_input("Keep? (y/n)")
if keep == 'y' or keep == "":
#record centroid position
x_cens.append(x_cen)
y_cens.append(y_cen)
#plot centroid position
plt.scatter(x_cen, y_cen, marker='+', c='r', s=80)
fig.canvas.draw()
if len(x_cens) == 1:
#First source -> prompt parameters
use_prev = 'n'
else:
#Prompt for previous parameter selection
use_prev = raw_input(
"Use previous photometry parameters? (y/n)")
if use_prev == 'y' or use_prev == "":
#Use previous parameters
apers.append(apers[-1])
psfs.append(psfs[-1])
fit_skys.append(fit_skys[-1])
else:
#Prompt for aperture size
aper = raw_input("Aperture=? (float / empty for PSF)")
if aper == "":
#PSF photometry parameters
psf = raw_input(
"PSF type=? ('1', '2', '3', 's<sersic index>')")
apers.append(None)
psfs.append(psf)
else:
#Aperture photometry parameters
aper = float(aper)
apers.append(aper)
psfs.append('1')
#Prompt for planar sky fitting
fit_sky = raw_input("Fit planar background? (y/n)")
if fit_sky == 'y' or fit_sky == "":
fit_skys.append(True)
else:
fit_skys.append(False)
else:
#Drop clicked position and keep listening
print "Dropping"
print ""
print "Click another source"
#plot image and listen for a click
cid = fig.canvas.mpl_connect('button_press_event', onclick)
if verbosity > 0:
print ""
print "Click on the sources"
print '---------------------------------------'
plt.show()
#convert source centroid positions to ra, dec degrees
x_cens = np.array(x_cens)
y_cens = np.array(y_cens)
ra_cens, dec_cens = wcs.all_pix2world(x_cens, y_cens, 0)
#compute magnitude of each source
RAs, DECs = np.zeros(len(x_cens)), np.zeros(len(x_cens))
Is, SNs = np.zeros(len(x_cens)), np.zeros(len(x_cens))
Ms, Merrs = np.zeros(len(x_cens)), np.zeros(len(x_cens))
if limsnr != 0:
#also compute limiting magnitudes
Mlims = np.zeros(len(x_cens))
for i in range(len(ra_cens)):
RA, DEC, I, SN, M, Merr, Mlim = magnitude(
image, image, wcs, cat, catname, (ra_cens[i], dec_cens[i]),
radius, apers[i], psfs[i], "Source" + str(i), band, fwhm,
limsnr, satmag, refmag, fit_skys[i], satpix, verbosity)
#output position, magnitude
print "Output for Source " + str(i)
print RA, DEC, I, SN, M, Merr, Mlim
RAs[i], DECs[i], Is[i], SNs[i], Ms[i], Merrs[i], Mlims[i] = RA[
0], DEC[0], I[0], SN[0], M[0], Merr[0], Mlim
retlist = [RAs, DECs, Is, SNs, Ms, Merrs, Mlims]
else:
#don't compute limiting magnitudes
for i in range(len(ra_cens)):
RA, DEC, I, SN, M, Merr = magnitude(
image, image, wcs, cat, catname, (ra_cens[i], dec_cens[i]),
radius, apers[i], psfs[i], "Source" + str(i), band, fwhm,
limsnr, satmag, refmag, fit_skys[i], satpix, verbosity)
#output position, magnitude
print "Output for Source " + str(i)
print RA, DEC, I, SN, M, Merr
RAs[i], DECs[i], Is[i], SNs[i], Ms[i], Merrs[i] = RA[0], DEC[0], I[
0], SN[0], M[0], Merr[0]
retlist = [RAs, DECs, Is, SNs, Ms, Merrs]
#plot diagnostic
if verbosity > 0:
#plot image of field
fig = plt.figure()
plt.imshow(image, cmap='Greys', vmax=0.0001 * np.amax(image), vmin=0)
plt.scatter(catX, catY, c='b', marker='+', label="catalog")
plt.scatter(x_cens, y_cens, marker='+', c='r', s=80, label="centroids")
x_meas, y_meas = wcs.all_world2pix(RAs, DECs, 0)
plt.scatter(x_meas, y_meas, marker='+', c='g', s=80, label="measured")
print len(x_cens), len(x_meas)
plt.tight_layout()
plt.show()
#return source measurements
return retlist
def show_tables():
Catalog.show_tables()
class FileSystem(Fuse):
null_stat = fuse.Stat(
st_mode=stat.S_IFDIR | 0755,
st_ino=0,
st_dev=0,
st_nlink=2,
st_uid=0,
st_gid=0,
st_size=0,
st_atime=0,
st_mtime=0,
st_ctime=0,
st_blksize=0,
st_rdev=0)
bacula_stat_fields = [
'st_dev', 'st_ino', 'st_mode', 'st_nlink', 'st_uid', 'st_gid',
'st_rdev', 'st_size', 'st_blksize', 'st_blocks', 'st_atime',
'st_mtime', 'st_ctime', 'st_linkfi', 'st_flags', 'st_streamid'
]
fuse_stat_fields = [
attr for attr in dir(null_stat) if attr.startswith('st_')
]
xattr_prefix = 'user.baculafs.'
xattr_fields = ['FileIndex', 'JobId', 'LStat', 'MD5']
xattr_fields_root = [
'client', 'fileset', 'datetime', 'joblist', 'cache_prefix'
]
xattr_fields_bextract = [
'path', 'volume', 'retries', 'state', 'pending', 'failures'
]
bextract_done = {
'path': None,
'volume': None,
'retries': 0,
'state': 'idle'
}
def __init__(self, *args, **kw):
'''
Initialize filesystem
'''
self._extract_lock = threading.Lock()
self._getattr_lock = threading.Lock()
self._bextract_status_lock = threading.Lock()
self._bextract_user_intervention_event = threading.Event()
self._initialized = False
# default option values
self.logging = 'info'
self.syslog = False
self.driver = SQL.SQLITE3
self.database = None
self.host = 'localhost'
self.port = 0
self.username = '******'
self.password = None
self.conf = '/etc/bacula/bacula-sd.conf'
self.client = ''
self.fileset = None
self.device = 'FileStorage'
self.datetime = None
self.recent_job = False
self.joblist = None
self.cache_prefix = None
self.user_cache_path = None
self.cleanup = False
self.move_root = False
self.prefetch_attrs = False
self.prefetch_regex = None
self.prefetch_symlinks = False
self.prefetch_recent = False
self.prefetch_diff = None
self.prefetch_difflist = None
self.prefetch_list = None
self.prefetch_everything = False
self.batch_mode = False
self.batch_list = False
self.batch_bsr = False
self.batch_extract = False
self.use_ino = False
self.max_ino = 0
self.dirs = {'/': {'': (FileSystem.null_stat, )}}
self._bextract_status = copy.deepcopy(FileSystem.bextract_done)
self._bextract_status['pending'] = 0
self._bextract_status['failures'] = 0
class File(FileSystem._File):
def __init__(self2, *a, **kw):
FileSystem._File.__init__(self2, self, *a, **kw)
self.file_class = File
Fuse.__init__(self, *args, **kw)
def _split(self, path):
'''
os.path.split wrapper
'''
head, tail = os.path.split(path)
if head and not head.endswith('/'):
head += '/'
return head, tail
def _bacula_stat(self, base64):
'''
Parse base64 encoded lstat info.
Returns fuse.Stat object with subset of decoded values,
and dictionary with full list of decoded values
'''
st = fuse.Stat()
lst = dict(
zip(FileSystem.bacula_stat_fields,
map(self.base64.decode, base64.split())))
for k in FileSystem.bacula_stat_fields:
if k in FileSystem.fuse_stat_fields:
setattr(st, k, lst[k])
return lst, st
def _add_parent_dirs(self, path):
'''
add parent directories of path to dirs dictionary
'''
head, tail = self._split(path[:-1])
if not head or head == path:
return
if not head in self.dirs:
self.dirs[head] = {tail: (FileSystem.null_stat, )}
elif not tail in self.dirs[head]:
self.dirs[head][tail] = (FileSystem.null_stat, )
self._add_parent_dirs(head)
def _update_inodes(self, head):
'''
generate unique st_ino for each missing st_ino
'''
for tail in self.dirs[head]:
if self.dirs[head][tail][-1].st_ino == 0:
if len(self.dirs[head][tail]) == 1:
self.dirs[head][tail] = \
(copy.deepcopy(FileSystem.null_stat),)
self.max_ino += 1
self.dirs[head][tail][-1].st_ino = self.max_ino
subdir = '%s%s/' % (head, tail)
if subdir in self.dirs:
self._update_inodes(subdir)
def _extract(self, path_list):
'''
extract path list from storage, returns path list of extracted files
'''
nitems = len(path_list)
self._bextract_increment_counter('pending', nitems)
# serialize extractions
self._extract_lock.acquire()
items = []
realpath_list = []
hardlink_targets = []
for path in path_list:
realpath, symlinkinfo, volumes = self._find_volumes(path)
realpath_list.append(realpath)
if volumes:
items.append((symlinkinfo, path, volumes))
# collect hard link targets
hardlink_target = self._hardlink_target(path)
if (hardlink_target and hardlink_target not in path_list
and hardlink_target not in hardlink_targets):
hardlink_targets.append(hardlink_target)
# add hardlink targets to list
# (bextract will fail to extract the hardlink
# if its target does not exist)
for path in hardlink_targets:
realpath, symlinkinfo, volumes = self._find_volumes(path)
if volumes:
items.append((symlinkinfo, path, volumes))
if len(items) > 0:
rc, sig = self._bextract(items)
# it seems that bextract does not restore mtime for symlinks
# so we create a normal file with same mtime as stored symlink
if rc == 0 and not self.batch_mode:
for item in items:
if item[0]:
symlinkfile = item[0][0]
symlinktime = item[0][1:]
makedirs(os.path.dirname(symlinkfile))
touch(symlinkfile, symlinktime)
self._extract_lock.release()
self._bextract_increment_counter('pending', -nitems)
return realpath_list
def _hardlink_target(self, path):
'''
return hard link target of path if it is a hard link
'''
head, tail = self._split(path)
bs = self.dirs[head][tail][-2]
jobid = self.dirs[head][tail][1]
if bs['st_nlink'] > 1 and bs['st_linkfi'] > 0:
st_linkfi = bs['st_linkfi']
for file in self.catalog.files:
if jobid == file[3] and st_linkfi == file[2]:
hardlink_target = ('/' if not file[0].startswith('/') else
'') + file[0] + file[1]
return hardlink_target
return None
def _find_volumes(self, path):
'''
return list of volumes that contain path to be extracted,
if the path has not been extracted yet
'''
realpath = os.path.normpath(self.cache_path + path)
symlinkpath = os.path.normpath(self.cache_symlinks + path)
head, tail = self._split(path)
# sanity check: path should not be a directory
if tail == '':
raise RuntimeError('trying to extract a directory %s' % path)
# check that path exists in catalog
if head not in self.dirs or tail not in self.dirs[head]:
return None, None, None
# sanity check: path entry is incomplete
if len(self.dirs[head][tail]) == 1:
raise RuntimeError('incomplete entry for path %s' % path)
# return if file has already been extracted
bs = self.getattr(path)
is_symlink = stat.S_ISLNK(bs.st_mode)
found = False
if os.path.exists(realpath) or os.path.lexists(realpath):
# make sure that stat info of realpath matches path
s = os.lstat(realpath)
conds = [
getattr(s, attr) == getattr(bs, attr) for attr in
['st_mode', 'st_uid', 'st_gid', 'st_size', 'st_mtime']
]
if is_symlink:
conds[-1] = (os.path.exists(symlinkpath)
and bs.st_mtime == os.stat(symlinkpath).st_mtime)
if all(conds):
return realpath, None, None
# generate list of volumes for path
fileindex, jobid = self.dirs[head][tail][0:2]
jobs = [job for job in self.catalog.jobs if job[0] == jobid]
volumes = [
[
volume[1], # 0-Volume
volume[2], # 1-MediaType
self.device, # 2-Device
jobs[0][0], # 3-JobId
jobs[0][1], # 4-VolSessionId
jobs[0][2], # 5-VolSessionTime
(volume[5] << 32) | volume[7], # 6-VolAddr: StartAddr
(volume[6] << 32) | volume[8], # 7-VolAddr: EndAddr
fileindex
] # 8-FileIndex
for volume in self.catalog.volumes
if (volume[0] == jobid and volume[3] <= fileindex
and fileindex <= volume[4])
]
return (realpath, ((symlinkpath, bs.st_atime,
bs.st_mtime) if is_symlink else None), volumes)
def _bextract_set_status(self, status):
'''
thread safe modification of bextract status dict
'''
self._bextract_status_lock.acquire()
for key in status:
self._bextract_status[key] = status[key]
self._bextract_status_lock.release()
def _bextract_increment_counter(self, counter, n):
'''
thread safe modification of bextract counters
'''
self._bextract_status_lock.acquire()
self._bextract_status[counter] += n
self._bextract_status_lock.release()
def _bextract_get_status(self):
'''
thread safe access to bextract status dict
'''
self._bextract_status_lock.acquire()
status = copy.deepcopy(self._bextract_status)
self._bextract_status_lock.release()
return status
def _bextract_flock(self):
'''
lock the storage daemon configuration file
'''
# we allow locking to fail, so as to allow
# at least a single instance of baculafs,
# even if we can't lock the sd conf file
try:
f = open(self.conf, 'r')
fcntl.flock(f, fcntl.LOCK_EX)
return f
except:
self.logger.warning(traceback.format_exc())
return None
def _bextract_funlock(self, f):
'''
unlock the file f
'''
if not f:
return
try:
fcntl.flock(f, fcntl.LOCK_UN)
f.close()
except:
self.logger.warning(traceback.format_exc())
def _bextract(self, items):
'''
extract list of items from Bacula storage device
'''
if self.batch_list:
for item in items:
print item[1]
if (not self.batch_bsr and not self.batch_extract):
return (0, 0)
bsrpath = self._write_bsr(items)
if self.batch_bsr:
bsrfile = open(bsrpath, 'rt')
for line in bsrfile:
sys.stdout.write(line)
sys.stdout.flush()
bsrfile.close()
if not self.batch_extract:
return (0, 0)
if self.batch_extract:
makedirs(self.fuse_args.mountpoint)
cmd = ('bextract -b "%s" -c "%s" "%s" "%s"' %
(bsrpath,
self.conf,
self.device,
self.cache_path \
if not self.batch_extract else self.fuse_args.mountpoint))
self.logger.debug(cmd)
self._bextract_set_status({
'path': items[0][1],
'volume': items[0][-1][0][0],
'retries': 0,
'state': 'run'
})
# we serialize calls to bextract across instances of baculafs
# by locking the storage daemon configuration file
# (note that this may not work over NFS)
f = self._bextract_flock()
child = pexpect.spawn(cmd)
child.logfile = self.logfile # sys.stdout
attempt = 0
missing = ''
while True:
# bextract either finishes or waits for a missing volume
i = child.expect([self.fail_pattern, pexpect.EOF],
timeout=None,
searchwindowsize=200)
self.logfile.flush(flush_tail=True)
if i == 0:
# count retries
if missing == child.match.groups()[0]:
attempt += 1
self._bextract_set_status({
'retries':
attempt,
'state':
'*user intervention required*'
})
else:
attempt = 1
missing = child.match.groups()[0]
self._bextract_set_status({
'volume':
missing,
'retries':
attempt,
'state':
'*user intervention required*'
})
# wait for user
if not self._initialized:
if self.loglevel != logging.DEBUG:
sys.stdout.write(
'Mount Volume "%s" on device "%s" %s '
'and press return when ready: ' %
(missing, self.device, child.match.groups()[1]))
sys.stdout.flush()
sys.stdin.read(1)
else:
self.logger.error(
'Mount volume "%s" on device "%s" %s and run '
'"attr -s baculafs.bextract.state -V run %s"'
' when ready' % (missing, self.device,
child.match.groups()[1],
self.fuse_args.mountpoint))
self._bextract_user_intervention_event.clear()
self._bextract_user_intervention_event.wait()
self._bextract_user_intervention_event.clear()
# retry
self._bextract_set_status({'state': 'run'})
child.sendline('')
else:
child.close()
break
# unlock the sd configuration file
self._bextract_funlock(f)
self._bextract_set_status(FileSystem.bextract_done)
if child.exitstatus or child.signalstatus:
self.logger.error('extraction failed (bsr file: %s)' % bsrpath)
self._bextract_increment_counter('failures', 1)
return (child.exitstatus, child.signalstatus)
def _group_by_volume(self, items):
'''
return items grouped by volume
'''
# group volumes
volumes = []
for item in items:
for v in item[-1]:
found = False
findex = v[-1]
for vindex in xrange(0, len(volumes)):
volume = volumes[vindex]
if not any(map(cmp, v[:-1], volume[:-1])):
volume[-1].append(findex)
found = True
break
if not found:
volumes.append(v[:-1] + [[v[-1]]])
# compact list of file indices
for volume in volumes:
volume[-1] = list(set(volume[-1]))
volume[-1].sort()
l = len(volume[-1])
findex = volume[-1][0]
findices = [(findex, findex)]
for idx in volume[-1][1:]:
next_idx = findices[-1][-1] + 1
if idx == next_idx:
findices[-1] = (findices[-1][0], idx)
else:
findices.append((idx, idx))
volume[-1] = findices
volume.append(l)
# reorder volumes to ensure correct handling of
# files spanning multiple volumes
volumes.sort(cmp=lambda a, b: \
(cmp(a[3], b[3]) or
cmp(a[8][-1][-1], b[8][0][0])))
return volumes
def _write_bsr(self, items):
'''
generate bsr for items to be extracted
'''
bsrfd, bsrpath = tempfile.mkstemp(
suffix='.bsr', dir=self.cache_bsrpath, text=True)
volumes = self._group_by_volume(items)
for volume in volumes:
os.write(bsrfd, 'Volume="%s"\n' % volume[0])
os.write(bsrfd, 'MediaType="%s"\n' % volume[1])
os.write(bsrfd, 'Device="%s"\n' % volume[2])
os.write(bsrfd, 'VolSessionId=%d\n' % volume[4])
os.write(bsrfd, 'VolSessionTime=%d\n' % volume[5])
if not self.bsr_compat:
os.write(bsrfd, 'VolAddr=%d-%d\n' % (volume[6], volume[7]))
for findex in volume[8]:
if findex[0] == findex[1]:
os.write(bsrfd, 'FileIndex=%d\n' % findex[0])
else:
os.write(bsrfd, 'FileIndex=%d-%d\n' % findex)
os.write(bsrfd, 'Count=%d\n' % volume[9])
os.close(bsrfd)
return bsrpath
def _match_stat(self, path, bs):
'''
determine if stat of path matches bs
'''
found = False
if os.path.exists(path) or os.path.lexists(path):
s = os.lstat(path)
found = all([
getattr(s, attr) == getattr(bs, attr) for attr in
['st_mode', 'st_uid', 'st_gid', 'st_size', 'st_mtime']
])
return found
def _setup_logging(self):
'''
initialize logging facility
'''
# log messages are sent to both console and syslog
# use -o logging=level to set the log level
# use -o syslog to enable logging to syslog
self.logger = logging.getLogger('BaculaFS')
self.loglevel = LOGGING_LEVELS.get(self.logging, logging.NOTSET)
self.logger.setLevel(self.loglevel)
h = logging.StreamHandler()
h.setLevel(self.loglevel)
formatter = logging.Formatter("%(message)s")
h.setFormatter(formatter)
self.logger.addHandler(h)
if self.syslog:
try:
h = logging.handlers.SysLogHandler('/dev/log')
h.setLevel(self.loglevel)
formatter = logging.Formatter(
"%(name)s: %(levelname)-8s - %(message)s")
h.setFormatter(formatter)
self.logger.addHandler(h)
except:
self.logger.warning(traceback.format_exc())
self.logfile = LogFile(self.logger, logging.DEBUG)
def initialize(self, version):
'''
initialize database, catalog
'''
self._setup_logging()
# batch mode
self.batch_mode = (self.batch_list or self.batch_bsr
or self.batch_extract)
# disable INFO level logging in batch mode
if self.batch_mode and self.loglevel == logging.INFO:
self.loglevel = logging.WARNING
self.logger.setLevel(self.loglevel)
self.logger.info('Populating file system ... ')
# setup cache
if self.user_cache_path:
self.cache_prefix = self.user_cache_path
else:
self.cache_prefix = tempfile.mkdtemp(prefix='baculafs-')
self.cache_path = os.path.normpath(self.cache_prefix + '/files')
makedirs(self.cache_path)
self.cache_bsrpath = os.path.normpath(self.cache_prefix + '/bsr')
makedirs(self.cache_bsrpath)
self.cache_symlinks = os.path.normpath(self.cache_prefix + '/symlinks')
makedirs(self.cache_symlinks)
# test for old version (2.x) of bacula
self.bsr_compat = int(version[0]) < 3
if self.bsr_compat:
self.logger.debug('Detected old Bacula: %s' % version)
# test access to sd conf file
open(self.conf, 'r').close()
# init bextract failure pattren
self.fail_pattern = ('Mount Volume "([^"]+)" on device "%s" (.*) '
'and press return when ready:' % self.device)
# init database and catalog
self.db = Database(self.driver, self.host, self.port, self.database,
self.username, self.password, self.logger)
self.catalog = Catalog(self.db)
self.base64 = Base64()
files = self.catalog.query(self.client, self.fileset, self.datetime,
self.recent_job, self.joblist)
# validated values
self.client = self.catalog.client
self.fileset = self.catalog.fileset[1]
self.datetime = self.catalog.datetime
# we don't need the database anymore
self.db.close()
prefetches = []
difflist = {}
# validate prefetch conditions
if self.prefetch_everything:
self.prefetch_recent = False
self.prefetch_regex = None
self.prefetch_diff = None
self.prefetch_difflist = None
self.prefetch_list = None
self.prefetch_symlinks = True
if self.prefetch_regex:
try:
regex = re.compile(self.prefetch_regex)
self.prefetch_attrs = True
except:
# bad regex: show traceback and ignore
self.logger.warning(traceback.format_exc())
self.prefetch_regex = None
if self.prefetch_diff:
self.prefetch_diff = os.path.normpath(
os.path.expanduser(self.prefetch_diff))
try:
if os.path.isdir(self.prefetch_diff):
self.prefetch_symlinks = True
else:
self.prefetch_diff = None
except:
# can't access target directory: show traceback and ignore
self.logger.warning(traceback.format_exc())
self.prefetch_diff = None
if self.prefetch_difflist:
self.prefetch_difflist = os.path.normpath(
os.path.expanduser(self.prefetch_difflist))
try:
difflistfile = (sys.stdin if self.prefetch_difflist == '-' else
open(self.prefetch_difflist, 'rt'))
for line in difflistfile.readlines():
date = ' '.join(line.split()[:5])
difflist[line[(len(date) + 1):].strip()] = \
time.strptime(date, '%a %b %d %H:%M:%S %Y')
difflistfile.close()
self.prefetch_symlinks = True
except:
# can't access/parse difflist: show traceback and ignore
self.logger.warning(traceback.format_exc())
self.prefetch_difflist = None
if self.prefetch_list:
self.prefetch_list = os.path.normpath(
os.path.expanduser(self.prefetch_list))
try:
listfile = (sys.stdin if self.prefetch_list == '-' else open(
self.prefetch_list, 'rt'))
matchlist = [line.strip() for line in listfile.readlines()]
listfile.close()
self.prefetch_symlinks = True
except:
# can't access/parse list: show traceback and ignore
self.logger.warning(traceback.format_exc())
self.prefetch_list = None
if self.prefetch_recent:
self.prefetch_symlinks = True
if self.prefetch_symlinks:
self.prefetch_attrs = True
if 'use_ino' in self.fuse_args.optlist:
self.use_ino = True
self.prefetch_attrs = True # must figure out max st_ino
for file in files:
head = file[0]
tail = file[1]
# handle windows directories
if not head.startswith('/'):
head = '/' + head
# make file entry
if self.prefetch_attrs:
entry = file[2:] + self._bacula_stat(file[-2])
# find max st_ino
if self.use_ino:
if entry[-1].st_ino > self.max_ino:
self.max_ino = entry[-1].st_ino
# detemine if we need to prefetch this entry
filepath = head + tail
if (not stat.S_ISDIR(entry[-1].st_mode) and
(self.prefetch_everything or
(self.prefetch_recent and
file[3] == self.catalog.most_recent_jobid) or
(self.prefetch_regex and
regex.search(filepath)) or
(self.prefetch_diff and
not self._match_stat(self.prefetch_diff + filepath,
entry[-1])) or
(self.prefetch_difflist and
(filepath[1:] not in difflist or
difflist[filepath[1:]][:-1] != \
time.localtime(entry[-1].st_mtime)[:-1])) or
(self.prefetch_list and
filepath in matchlist) or
(self.prefetch_symlinks and
stat.S_ISLNK(entry[-1].st_mode)))):
prefetches.append(filepath)
else:
entry = file[2:] + (None, ) # stat info placeholder
# new directory
if head not in self.dirs:
self.dirs[head] = {}
# add parent directories
self._add_parent_dirs(head)
# directories are added to their parents
if head != '/' and tail == '':
head, tail = self._split(head[:-1])
# and finally
self.dirs[head][tail] = entry
# fix st_ino
if self.use_ino:
self._update_inodes('/')
npf = len(prefetches)
if npf > 0:
self.logger.info('Prefetching %d objects ... ' % npf)
self._extract(prefetches)
self.logger.debug('Cache directory is: %s' % self.cache_prefix)
self.joblist = ' '.join([str(job[0]) for job in self.catalog.jobs])
self.logger.debug('Job ids in file system: %s' % self.joblist)
self.logger.info('BaculaFS ready (%d files).' % len(files))
self._initialized = True
def shutdown(self):
'''
remove cache directory if required
'''
if self.cleanup and not self.user_cache_path and self.cache_prefix:
self.logger.info(
'removing cache directory: %s' % self.cache_prefix)
shutil.rmtree(self.cache_prefix, ignore_errors=True)
def setxattr(self, path, name, value, flags):
'''
set value of extended attribute
(we only allow setting user.baculafs.bextract.state on
the root directory)
'''
if (path == '/' and name == FileSystem.xattr_prefix + 'bextract.state'
and value == 'run'):
self._bextract_user_intervention_event.set()
else:
return -errno.EOPNOTSUPP
def getxattr(self, path, name, size):
'''
get value of extended attribute
baculafs exposes some filesystem attributes for the root directory
(e.g. joblist, cache_prefix - see FileSystem.xattr_fields_root)
and several other attributes for each file/directory that appears
in the catalog (e.g. MD5, JobId - see FileSystem.xattr_fields)
'''
head, tail = self._split(path)
val = None
n = name.replace(FileSystem.xattr_prefix, '')
if path == '/':
if n in FileSystem.xattr_fields_root:
val = str(getattr(self, n))
elif n.startswith('bextract.'):
n = n.replace('bextract.', '')
if n in FileSystem.xattr_fields_bextract:
val = str(self._bextract_get_status()[n])
if (not val and head in self.dirs and tail in self.dirs[head]
and len(self.dirs[head][tail]) != 1
and n in FileSystem.xattr_fields):
val = str(self.dirs[head][tail][FileSystem.xattr_fields.index(n)])
if n == 'MD5' and val != '0':
l = len(val)
val = binascii.b2a_hex(
binascii.a2b_base64(val + '=' * (
(l * 3 + 8) / 3 - l) + '\n')) # padding
# attribute not found
if val == None:
return -errno.ENODATA
# We are asked for size of the value.
if size == 0:
return len(val)
return val
def listxattr(self, path, size):
'''
list extended attributes
'''
head, tail = self._split(path)
xattrs = []
if path == '/':
xattrs += [
FileSystem.xattr_prefix + a
for a in FileSystem.xattr_fields_root
]
xattrs += [
FileSystem.xattr_prefix + 'bextract.' + a
for a in FileSystem.xattr_fields_bextract
]
if (head in self.dirs and tail in self.dirs[head]
and len(self.dirs[head][tail]) != 1):
xattrs += [
FileSystem.xattr_prefix + a for a in FileSystem.xattr_fields
]
# We are asked for size of the attr list, ie. joint size of attrs
# plus null separators.
if size == 0:
return len("".join(xattrs)) + len(xattrs)
return xattrs
def getattr(self, path):
'''
Retrieve file attributes.
Notes:
1) Bacula does not store attributes for parent directories
that are not being explicitly backed up, so we provide
a default set of attributes FileSystem.null_stat
2) file attributes are base64-encoded and stored by Bacula
in the catalog. These attributes are decoded when first
needed and then cached for subsequent requests.
3) python fuse expects atime/ctime/mtime to be positive
'''
head, tail = self._split(path)
if head in self.dirs and tail in self.dirs[head]:
self._getattr_lock.acquire()
attrs = self.dirs[head][tail][-1]
# decode and cache stat info
if not attrs:
self.dirs[head][tail] = self.dirs[head][tail][:-1] + \
self._bacula_stat(self.dirs[head][tail][-3])
attrs = self.dirs[head][tail][-1]
# zero negative timestamps
for a in ['st_atime', 'st_mtime', 'st_ctime']:
t = getattr(attrs, a)
if t < 0:
self.logger.warning(
'%s has negative timestamp %s=%d, will use 0' % (path,
a, t))
setattr(attrs, a, 0)
self._getattr_lock.release()
return attrs
else:
return -errno.ENOENT
def readdir(self, path, offset):
'''
read directory entries
'''
path = path if path.endswith('/') else path + '/'
for key in ['.', '..']:
yield fuse.Direntry(key)
for key in self.dirs[path].keys():
if len(key) > 0:
if self.use_ino:
bs = self.getattr(path + key)
ino = bs.st_ino
else:
ino = 0
yield fuse.Direntry(key, ino=ino)
def readlink(self, path):
'''
read link contents
'''
realpath = self._extract([path])[0]
if realpath:
link = os.readlink(realpath)
if self.move_root and link.startswith('/'):
link = os.path.normpath(self.fuse_args.mountpoint + link)
return link
return -errno.ENOENT
class _File(object):
def __init__(self, fs, path, flags, *mode):
self.fs = fs
accmode = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
if (flags & accmode) != os.O_RDONLY:
raise IOError(errno.EACCES, '')
self.path = path
self.realpath = fs._extract([path])[0]
self.file = os.fdopen(
os.open(self.realpath, flags, *mode), flag2mode(flags))
self.fd = self.file.fileno()
self.direct_io = False
self.keep_cache = True
def read(self, length, offset):
self.file.seek(offset)
return self.file.read(length)
def release(self, flags):
self.file.close()
def initialize(path: str):
Catalog.__initialize__(path)
Index.__initialize__(path)
Buffer.__initialize__()
def initialize(self, version):
'''
initialize database, catalog
'''
self._setup_logging()
# batch mode
self.batch_mode = (self.batch_list or self.batch_bsr
or self.batch_extract)
# disable INFO level logging in batch mode
if self.batch_mode and self.loglevel == logging.INFO:
self.loglevel = logging.WARNING
self.logger.setLevel(self.loglevel)
self.logger.info('Populating file system ... ')
# setup cache
if self.user_cache_path:
self.cache_prefix = self.user_cache_path
else:
self.cache_prefix = tempfile.mkdtemp(prefix='baculafs-')
self.cache_path = os.path.normpath(self.cache_prefix + '/files')
makedirs(self.cache_path)
self.cache_bsrpath = os.path.normpath(self.cache_prefix + '/bsr')
makedirs(self.cache_bsrpath)
self.cache_symlinks = os.path.normpath(self.cache_prefix + '/symlinks')
makedirs(self.cache_symlinks)
# test for old version (2.x) of bacula
self.bsr_compat = int(version[0]) < 3
if self.bsr_compat:
self.logger.debug('Detected old Bacula: %s' % version)
# test access to sd conf file
open(self.conf, 'r').close()
# init bextract failure pattren
self.fail_pattern = ('Mount Volume "([^"]+)" on device "%s" (.*) '
'and press return when ready:' % self.device)
# init database and catalog
self.db = Database(self.driver, self.host, self.port, self.database,
self.username, self.password, self.logger)
self.catalog = Catalog(self.db)
self.base64 = Base64()
files = self.catalog.query(self.client, self.fileset, self.datetime,
self.recent_job, self.joblist)
# validated values
self.client = self.catalog.client
self.fileset = self.catalog.fileset[1]
self.datetime = self.catalog.datetime
# we don't need the database anymore
self.db.close()
prefetches = []
difflist = {}
# validate prefetch conditions
if self.prefetch_everything:
self.prefetch_recent = False
self.prefetch_regex = None
self.prefetch_diff = None
self.prefetch_difflist = None
self.prefetch_list = None
self.prefetch_symlinks = True
if self.prefetch_regex:
try:
regex = re.compile(self.prefetch_regex)
self.prefetch_attrs = True
except:
# bad regex: show traceback and ignore
self.logger.warning(traceback.format_exc())
self.prefetch_regex = None
if self.prefetch_diff:
self.prefetch_diff = os.path.normpath(
os.path.expanduser(self.prefetch_diff))
try:
if os.path.isdir(self.prefetch_diff):
self.prefetch_symlinks = True
else:
self.prefetch_diff = None
except:
# can't access target directory: show traceback and ignore
self.logger.warning(traceback.format_exc())
self.prefetch_diff = None
if self.prefetch_difflist:
self.prefetch_difflist = os.path.normpath(
os.path.expanduser(self.prefetch_difflist))
try:
difflistfile = (sys.stdin if self.prefetch_difflist == '-' else
open(self.prefetch_difflist, 'rt'))
for line in difflistfile.readlines():
date = ' '.join(line.split()[:5])
difflist[line[(len(date) + 1):].strip()] = \
time.strptime(date, '%a %b %d %H:%M:%S %Y')
difflistfile.close()
self.prefetch_symlinks = True
except:
# can't access/parse difflist: show traceback and ignore
self.logger.warning(traceback.format_exc())
self.prefetch_difflist = None
if self.prefetch_list:
self.prefetch_list = os.path.normpath(
os.path.expanduser(self.prefetch_list))
try:
listfile = (sys.stdin if self.prefetch_list == '-' else open(
self.prefetch_list, 'rt'))
matchlist = [line.strip() for line in listfile.readlines()]
listfile.close()
self.prefetch_symlinks = True
except:
# can't access/parse list: show traceback and ignore
self.logger.warning(traceback.format_exc())
self.prefetch_list = None
if self.prefetch_recent:
self.prefetch_symlinks = True
if self.prefetch_symlinks:
self.prefetch_attrs = True
if 'use_ino' in self.fuse_args.optlist:
self.use_ino = True
self.prefetch_attrs = True # must figure out max st_ino
for file in files:
head = file[0]
tail = file[1]
# handle windows directories
if not head.startswith('/'):
head = '/' + head
# make file entry
if self.prefetch_attrs:
entry = file[2:] + self._bacula_stat(file[-2])
# find max st_ino
if self.use_ino:
if entry[-1].st_ino > self.max_ino:
self.max_ino = entry[-1].st_ino
# detemine if we need to prefetch this entry
filepath = head + tail
if (not stat.S_ISDIR(entry[-1].st_mode) and
(self.prefetch_everything or
(self.prefetch_recent and
file[3] == self.catalog.most_recent_jobid) or
(self.prefetch_regex and
regex.search(filepath)) or
(self.prefetch_diff and
not self._match_stat(self.prefetch_diff + filepath,
entry[-1])) or
(self.prefetch_difflist and
(filepath[1:] not in difflist or
difflist[filepath[1:]][:-1] != \
time.localtime(entry[-1].st_mtime)[:-1])) or
(self.prefetch_list and
filepath in matchlist) or
(self.prefetch_symlinks and
stat.S_ISLNK(entry[-1].st_mode)))):
prefetches.append(filepath)
else:
entry = file[2:] + (None, ) # stat info placeholder
# new directory
if head not in self.dirs:
self.dirs[head] = {}
# add parent directories
self._add_parent_dirs(head)
# directories are added to their parents
if head != '/' and tail == '':
head, tail = self._split(head[:-1])
# and finally
self.dirs[head][tail] = entry
# fix st_ino
if self.use_ino:
self._update_inodes('/')
npf = len(prefetches)
if npf > 0:
self.logger.info('Prefetching %d objects ... ' % npf)
self._extract(prefetches)
self.logger.debug('Cache directory is: %s' % self.cache_prefix)
self.joblist = ' '.join([str(job[0]) for job in self.catalog.jobs])
self.logger.debug('Job ids in file system: %s' % self.joblist)
self.logger.info('BaculaFS ready (%d files).' % len(files))
self._initialized = True
def __init__(self):
self.catalog = Catalog("Catalog.json", "Buyers.json", "configCatalog.json", "devices.json")
from Catalog import *
from User import *
from pdb import set_trace
catalog = Catalog()
# catalog.displayAllBooks()
# set_trace()
librarian = Librarian("Awantik", "Bangalore", 34, 'asljlkj22', 'zeke101')
librarian.catalogAccess(catalog)
book = librarian.addBook('Shoe Dog', 'Phil Knight', '2015', 312)
librarian.addBookItem(book, '123hg', 'H1B2')
librarian.addBookItem(book, '123hg', 'H1B4')
librarian.addBookItem(book, '123hg', 'H1B5')
book = catalog.addBook('Moonwalking with Einstien', 'J Foer', '2017', 318)
librarian.addBookItem(book, '463hg', 'K1B2')
librarian.addBookItem(book, '463hg', 'K1B3')
librarian.addBookItem(book, '463hg', 'K1B4')
# catalog.displayAllBooks()
m1 = Member("Vishal", "Bangalore", 23, 'asljlkj22', 'std1233')
# m1.availableBooks(catalog)
m1.issueBook(catalog, 'Moonwalking with Einstien', '463hg', 5)
# m1.availableBooks(catalog)
#
# m1.issued_books_details()
# m1.availableBooks(catalog)
# m1.returnBook(catalog,"Moonwalking with Einstien",'463hg','K1B4')
# m1.availableBooks(catalog)
