def getDataset(self, **kwargs):
'''Interactive method. It prints the datasets (the user can apply filters),
the user chooses one of them and inserts the number of LFNs he wants.'''
manager = SBDatasetManager.SBDatasetManager()
def validateFilter(filter, allowed):
kwargs[filter] = kwargs.get(filter, allowed)
if not isinstance(kwargs[filter], list):
kwargs[filter] = [kwargs[filter]]
if not set(kwargs[filter]).issubset(set(allowed)):
raise GangaException('%s must be %s' % (filter, allowed))
validateFilter('status', ['open', 'closed'])
validateFilter('session', ['analysis'])
kwargs['files'] = 0
datasets = manager.getDataset(**kwargs)
dataset = manager.printDatasets(datasets)
self.dataset_id = dataset['dataset_id']
print('\nChosen dataset details:')
manager.printDatasetDetail(dataset)
print(
'\nInsert the minimum number of files that you need for your analysis (zero for all):'
)
self.files_total = utils.getIndex(maxInclusive=int(dataset['files']))
lfns = self.__getLFNs()
tot_size = 0
tot_files = len(lfns)
for lfn in lfns:
tot_size += int(lfn['size'])
print('\nTotal job input size: ' +
str(utils.sizeof_fmt_binary(tot_size)))
print('Total number of involved lfns: ' + str(tot_files))
print(
'\nInsert the maximum number of files for each subjob. Remember:')
print('- maximum output size is 2GiB.')
print('- suggested maximum job duration 18h.')
print('- maximum input size job is 10GiB.')
self.files_per_subjobs = utils.getIndex(minInclusive=1,
maxInclusive=tot_files)
job = self.__createInputPath(lfns)
print('\nSubjobs details:')
column_names = ['id', 'list_path', 'size', 'lfns']
print(utils.format_dict_table(job, column_names))
def getDataset(self, **kwargs):
'''Interactive mathod. It prints the datasets (the user can apply filters),
the user chooses one of them and inserts the number of events he wants.'''
manager = SBDatasetManager.SBDatasetManager()
def validateFilter(filter, allowed):
kwargs[filter] = kwargs.get(filter, allowed)
if not isinstance(kwargs[filter], list):
kwargs[filter] = [kwargs[filter]]
if not set(kwargs[filter]).issubset(set(allowed)):
raise GangaException('%s must be %s' % (filter, allowed))
validateFilter('status', ['open', 'closed'])
validateFilter('session', ['fastsim', 'fullsim'])
datasets = manager.getDataset(**kwargs)
dataset = manager.printDatasets(datasets)
self.dataset_id = dataset['dataset_id']
print('\nChosen dataset details:')
manager.printDatasetDetail(dataset)
print('\nInsert the minimum number of events that you need for your analysis (zero for all):')
self.events_total = utils.getIndex(maxInclusive=int(dataset['parameters']['evt_tot']))
lfns = self.__getLFNs(dataset['parameters']['evt_file'])
tot_size = 0
tot_files = len(lfns)
tot_events = int(dataset['parameters']['evt_file']) * tot_files
for lfn in lfns:
tot_size += int(lfn['size'])
print('\nTotal job input size: ' + str(utils.sizeof_fmt_binary(tot_size)))
print('Total selected number of events: ' + str(utils.sizeof_fmt_decimal(tot_events)))
print('Total number of involved lfns: ' + str(tot_files))
print('\nInsert the maximum number of events for each subjob. Remember:')
print('- maximum output size is 2GiB.')
print('- suggested maximum job duration 18h.')
print('- maximum input size job is 10GiB.')
print('- at least %s (that is the number of events of one file).' % dataset['parameters']['evt_file'])
self.events_per_subjobs = utils.getIndex(minInclusive=int(dataset['parameters']['evt_file']), maxInclusive=tot_events)
job = self.__createInputPath(lfns, dataset['parameters']['evt_file'])
print('\nSubjobs details:')
column_names = ['id', 'list_path', 'size', 'events', 'lfns']
print(utils.format_dict_table(job, column_names))
def asksParameter(parameter):
'''Interactive method requesting user the value of each parameter
per session (FastSim, FullSim, Analysis)'''
if parameter['customValue'] and len(parameter['values']) == 0:
value = raw_input('\nEnter %s: ' % parameter['label'])
elif not parameter['customValue'] and len(parameter['values']) == 0:
raise GangaException('Invalid rule (customValue:False and values=0).')
else:
table = list()
i = 0
for value in parameter['values']:
table.append({'id': i, 'value': value})
i += 1
if parameter['customValue']:
table.append({'id': i, 'value': 'Enter a custom value'})
print('\nChoose %s:' % parameter['label'])
column_names = ('id', 'value')
print(utils.format_dict_table(table, column_names))
index = utils.getIndex(maxExclusive=len(table))
if parameter['customValue'] and index == len(table)-1:
value = raw_input('Custom value: ')
else:
value = table[index]['value']
# parameter insertion in dictionary. It will be subsequently
#inserted into dataset analysis bookkeeping table, hstore field
new_dataset['parameters'][parameter['name']] = value
return value
def asksParameter(parameter):
'''Interactive method requesting user the value of each parameter
per session (FastSim, FullSim, Analysis)'''
if parameter['customValue'] and len(parameter['values']) == 0:
value = raw_input('\nEnter %s: ' % parameter['label'])
elif not parameter['customValue'] and len(
parameter['values']) == 0:
raise GangaException(
'Invalid rule (customValue:False and values=0).')
else:
table = list()
i = 0
for value in parameter['values']:
table.append({'id': i, 'value': value})
i += 1
if parameter['customValue']:
table.append({'id': i, 'value': 'Enter a custom value'})
print('\nChoose %s:' % parameter['label'])
column_names = ('id', 'value')
print(utils.format_dict_table(table, column_names))
index = utils.getIndex(maxExclusive=len(table))
if parameter['customValue'] and index == len(table) - 1:
value = raw_input('Custom value: ')
else:
value = table[index]['value']
# parameter insertion in dictionary. It will be subsequently
#inserted into dataset analysis bookkeeping table, hstore field
new_dataset['parameters'][parameter['name']] = value
return value
def baseline(timestamps):
"""
Implements Baseline algorithm
Parameters
----------
timestamps : list of tuples
Sorted list of interactions [(t1, n1, n2), (t2, n3, n4),...], where t is timestamp, n1 and n2 are interactiong nodes.
Nodes in the interactions are sorted lexicographically.
Returns
-------
tuple of dicts
(dict1, dict2): dict1 of a shape {n1: t1, n2: t2, ..} contains starting point t1 of activity interval of node n1, only for nodes chosen to be active.
dict2 of a shape {n1: t1, n2: t2, ..} contains ending point t1 of activity interval of node n1, only for nodes chosen to be active.
"""
Xstart, Xend = {}, {}
while timestamps:
nodeEdgeIndex = utils.getIndex(timestamps)
gain = {
node: np.float64(len(tstmps)) / (tstmps[-1][0] - tstmps[0][0])
for node, tstmps in nodeEdgeIndex.iteritems()
}
node, gain = (sorted(gain.items(), key=operator.itemgetter(1)))[-1]
s = sorted([i[0] for i in nodeEdgeIndex[node]])
Xstart[node], Xend[node] = s[0], s[-1]
timestamps = [x for x in timestamps if x[1] != node and x[2] != node]
return Xstart, Xend
def getDH():
global g, p, a, b, A, B, K1, K2
simpes = utils.getPrime(0)
arr = utils.getIndex(simpes, 0)
i = random.randint(1, len(arr) - 1)
g = arr[i]
arr.pop(i)
p = arr[random.randint(1, len(arr) - 1)]
simpes = utils.getPrime(4096)
arr = utils.getIndex(simpes, 4096)
index = random.randint(1, len(arr) - 1)
a = arr[index]
arr.pop(index)
b = arr[random.randint(1, len(arr) - 1)]
A = pow(g, a, p)
B = pow(g, b, p)
K1 = pow(B, a, p)
K2 = pow(A, b, p)
return g, p, a, b, A, B, K1, K2
def index(request):
'''显示首页'''
# 定义模板上下文
request.session.set_expiry(0)
context = utils.getIndex()
# 使用模板
if 'username' in request.session:
logger.info(request.session["username"])
else:
logger.info('anonymous')
print(context)
return render(request, 'books/index.html', context)
def getRSA ():
simples = utils.getPrime(1048576)
arr = utils.getIndex(simples, 1048576)
p = arr[random.randint(1, len(arr) - 1)]
q = arr[random.randint(1, len(arr) - 1)]
n = p * q
predF = (p - 1) * (q - 1)
d = 0
while d == 0:
e = random.randint(1, predF)
try:
d = mod_inverse(e, predF)
except:
d = 0
d = int(d)
return d, e, n, predF
def runInner(timestamps, max_iter=10):
"""
Implements Inner algorithm
Parameters
----------
timestamps : list of tuples
Sorted list of interactions [(t1, n1, n2), (t2, n3, n4),...], where t is timestamp, n1 and n2 are interactiong nodes.
Nodes in the interactions are sorted lexicographically.
maxiter : int
maximum number of interactions
Returns
-------
tuple of dicts
(dict1, dict2): dict1 of a shape {n1: t1, n2: t2, ..} contains starting point t1 of activity interval of node n1, for every node,
dict2 of a shape {n1: t1, n2: t2, ..} contains ending point t1 of activity interval of node n1, for every node.
"""
nodeEdgeIndex = utils.getIndex(timestamps, 'inner')
m = getInitial(nodeEdgeIndex)
alpha, alphaNodes = getAlphas(timestamps, m)
Xstart, Xend = getSolution(alphaNodes, m)
best = utils.getCost(Xstart, Xend)
bestSol = (Xstart, Xend)
for k in xrange(1, max_iter):
m = getNextInput(nodeEdgeIndex, Xstart, Xend)
alpha, alphaNodes = getAlphas(timestamps, m)
Xstart, Xend = getSolution(alphaNodes, m)
t = utils.getCost(Xstart, Xend)
if t < best:
best = t
bestSol = (Xstart, Xend)
else:
break
return bestSol[0], bestSol[1]
def runBudget(timestamps, maxiter=10):
"""
Implements Budget algorithm
Parameters
----------
timestamps : list of tuples
Sorted list of interactions [(t1, n1, n2), (t2, n3, n4),...], where t is timestamp, n1 and n2 are interactiong nodes.
Nodes in the interactions are sorted lexicographically.
maxiter : int
maximum number of interactions in binary search
Returns
-------
tuple of dicts
(dict1, dict2): dict1 of a shape {n1: t1, n2: t2, ..} contains starting point t1 of activity interval of node n1, for every node,
dict2 of a shape {n1: t1, n2: t2, ..} contains ending point t1 of activity interval of node n1, for every node.
"""
nodeEdgeIndex = utils.getIndex(timestamps, 'budget')
maxb = timestamps[-1][0] - timestamps[0][0]
klow, kup = 1. / maxb, 1.0
b = {n: maxb for n in nodeEdgeIndex}
LXstart, LXend = budgetAlgorithm(timestamps, nodeEdgeIndex, b)
c = 1
alpha = 1.0 / maxb
while (kup - klow) > alpha and c < maxiter:
k = (kup + klow) / 2
b = {n: maxb * k for n in nodeEdgeIndex}
Xstart, Xend = budgetAlgorithm(timestamps, nodeEdgeIndex, b)
c += 1
uncovered = utils.checkCoverage(Xstart, Xend, timestamps)
if uncovered:
klow = k
else:
kup = k
LXstart, LXend = Xstart, Xend
return LXstart, LXend
def setSoftwareVersion(self):
'''Set software version. This function is called when interactive()
cannot find it automatically or is called by user if prefer set it
manually'''
results = db.read('''SELECT session_name, soft_version
FROM session_site_soft
GROUP BY session_name, soft_version
ORDER BY session_name, soft_version''')
i = 0
for result in results:
result['id'] = i
i += 1
print('Choose simulation type:')
column_names = ('id', 'session_name', 'soft_version')
print(utils.format_dict_table(results, column_names))
index = utils.getIndex(maxExclusive=len(results))
self.session = results[index]['session_name']
self.soft_version = results[index]['soft_version']
def setSoftwareVersion(self):
'''Set software version. This function is called when interactive()
cannot find it automatically or is called by user if prefer set it
manually'''
results = db.read('''SELECT session_name, soft_version
FROM session_site_soft
GROUP BY session_name, soft_version
ORDER BY session_name, soft_version''')
i = 0
for result in results:
result['id'] = i
i += 1
print('Choose simulation type:')
column_names = ('id', 'session_name', 'soft_version')
print(utils.format_dict_table(results, column_names))
index = utils.getIndex(maxExclusive=len(results))
self.session = results[index]['session_name']
self.soft_version = results[index]['soft_version']
default=0.5,
type=float)
parser.add_argument("--nnodes",
help="number of nodes in the graph",
default=100,
type=int)
args = parser.parse_args()
event_length = args.intlen
overlap = args.overlap
num_nodes = args.nnodes
G = utils.generateGraph(n=num_nodes)
timestamps, active_truth = utils.generateIntervals(
G, event_length=event_length, overlap=overlap)
nodeEdgeIndex = utils.getIndex(timestamps, 'inner')
Cost_IP, P_IP, R_IP, F_IP, Costmax_IP = [], [], [], [], []
iterations = range(1, 11)
for iter in iterations:
if iter == 1:
m = inner.getInitial(nodeEdgeIndex)
Xstart, Xend = inner.runInner_iteration(timestamps, nodeEdgeIndex,
m)
else:
m = inner.getNextInput(nodeEdgeIndex, Xstart, Xend)
Xstart, Xend = inner.runInner_iteration(timestamps, nodeEdgeIndex,
m)
def printDatasets(self, datasets):
''' Given the heterogeneous dataset list, the method splits it in
categories and build the table per session. A unique id all over
the sessions permit the user to select univocally a dataset. All
the metadata will be printed with the parent chain: a parent dataset
is defined as the one used as input to create the child one. The
method is public but not exported in GPI'''
# check the term width
(width, height) = utils.getTerminalSize()
if width < 160:
logger.error(
"Your terminal's width is %d; must be at least 160\nYou can make your font size smaller"
% width)
raise utils.QuitException()
# better a named tuple but available in Python 2.6 only
# fullsim: the table elements should be common to all the dataset keys in the list
# print table rules:
grouped_datasets = [
# 0
{
'title':
'Fastsim Official Production',
'dataset':
list(),
'order_by': [
'prod_series', 'analysis', 'generator', 'dg', 'bkg_mixing',
'analysis_type'
],
'columns': [
'id', 'prod_series', 'analysis', 'generator', 'dg',
'bkg_mixing', 'analysis_type', 'status'
]
},
# 1
{
'title':
'Fastsim Personal Production',
'dataset':
list(),
'order_by': [
'free_string', 'analysis', 'generator', 'dg', 'bkg_mixing',
'analysis_type'
],
'columns': [
'id', 'free_string', 'analysis', 'generator', 'dg',
'bkg_mixing', 'analysis_type', 'status'
],
},
# 2
{
'title':
'Fullsim Official Production',
'dataset':
list(),
'order_by': [
'prod_series', 'simtype', 'generator', 'dg', 'pl', 'g4ver',
'opt_photons'
],
'columns': [
'id', 'prod_series', 'simtype', 'generator', 'dg', 'pl',
'g4ver', 'opt_photons', 'status'
]
},
# 3
{
'title':
'Fullsim Personal Production',
'dataset':
list(),
'order_by': [
'free_string', 'generator', 'dg', 'pl', 'g4ver',
'opt_photons'
],
'columns': [
'id', 'free_string', 'generator', 'dg', 'pl', 'g4ver',
'opt_photons', 'status'
]
},
# 4
{
'title': 'Analysis',
'dataset': list(),
'order_by': ['free_string', 'creation_date'],
'columns': ['id', 'free_string', 'creation_date', 'status']
}
]
for dataset in datasets:
# put sub dictionary elements to level zero dictionary
for key, value in dataset.items():
if type(dataset[key]) is types.DictType:
for key1, value1 in dataset[key].iteritems():
dataset[key1] = value1
#del dataset[key]
# dataset selection
if dataset['session'] == 'fastsim':
if dataset['owner'] == 'Official':
grouped_datasets[0]['dataset'].append(dataset)
else:
grouped_datasets[1]['dataset'].append(dataset)
elif dataset['session'] == 'fullsim':
if dataset['owner'] == 'Official':
grouped_datasets[2]['dataset'].append(dataset)
else:
grouped_datasets[3]['dataset'].append(dataset)
elif dataset['session'] == 'analysis':
grouped_datasets[4]['dataset'].append(dataset)
else:
raise GangaException('session not recognized: %s' %
dataset['session'])
i = 0
# field sort, adding id and print
for group in grouped_datasets:
if len(group['dataset']) > 0:
print('\n%s' % group['title'])
# dictionary sorting
group['dataset'] = sorted(
group['dataset'],
key=lambda elem: ('%s ' * len(group['order_by'])) % tuple(
[elem[d] for d in group['order_by']]))
# id adding
for dataset in group['dataset']:
dataset['id'] = i
i += 1
print(
utils.format_dict_table(group['dataset'],
group['columns']))
# ask for input and print dataset details
if i == 1:
index = 0
print('\nAutomatically selected the only entry')
else:
print('\nChoose the dataset:')
index = utils.getIndex(maxExclusive=i)
# Object oriented solution to investigate and/or binary search
# datasets have been grouped per print rules
for group in grouped_datasets:
for d in group['dataset']:
if d['id'] == index:
for dataset in datasets:
if dataset['dataset_id'] == d['dataset_id']:
return dataset
def runBudget_fixed_budget(timestamps, b={}):
nodeEdgeIndex = utils.getIndex(timestamps, 'budget')
if not b:
b = {n: (timestamps[-1][0] - timestamps[0][0]) for n in nodeEdgeIndex}
Xstart, Xend = budgetAlgorithm(timestamps, nodeEdgeIndex, b)
return Xstart, Xend
def createDataset(self):
'''Interactive method to guide the user in dataset creation procedure.
If the dataset is a 'personal production' type, force user to provide
a filter key.'''
def asksParameter(parameter):
'''Interactive method requesting user the value of each parameter
per session (FastSim, FullSim, Analysis)'''
if parameter['customValue'] and len(parameter['values']) == 0:
value = raw_input('\nEnter %s: ' % parameter['label'])
elif not parameter['customValue'] and len(parameter['values']) == 0:
raise GangaException('Invalid rule (customValue:False and values=0).')
else:
table = list()
i = 0
for value in parameter['values']:
table.append({'id': i, 'value': value})
i += 1
if parameter['customValue']:
table.append({'id': i, 'value': 'Enter a custom value'})
print('\nChoose %s:' % parameter['label'])
column_names = ('id', 'value')
print(utils.format_dict_table(table, column_names))
index = utils.getIndex(maxExclusive=len(table))
if parameter['customValue'] and index == len(table)-1:
value = raw_input('Custom value: ')
else:
value = table[index]['value']
# parameter insertion in dictionary. It will be subsequently
#inserted into dataset analysis bookkeeping table, hstore field
new_dataset['parameters'][parameter['name']] = value
return value
type = [
dict(id = 0, dataset_type = 'FastSim Personal Production'),
dict(id = 1, dataset_type = 'FullSim Personal Production'),
dict(id = 2, dataset_type = 'Analysis'),
]
column_names = ('id', 'dataset_type')
print(utils.format_dict_table(type, column_names))
index = utils.getIndex(maxExclusive=len(type))
new_dataset = dict()
new_dataset['parameters'] = dict()
####################
# FAST Simulation session
####################
# parameter check: mandatory, free string param management
# TODO: parameter type check, evaluate the config file option to store parameters
if index == 0:
new_dataset['session'] = 'fastsim'
parameters = [
{"name": "evt_file", "label": "Events per file", "customValue": True, "values": []},
{"name": "analysis", "label": "Analysis", "customValue": True, "values": ["BtoKNuNu", "BtoKstarNuNu", "DstD0ToXLL", "DstD0ToXLL", "Generics", "HadRecoilCocktail", "KplusNuNu", "SLRecoilCocktail", "tau->3mu"]},
{"name": "dg", "label": "Geometry", "customValue": True, "values": ["DG_4", "DG_4a", "DG_BaBar"]},
{"name": "generator", "label": "Generator", "customValue": True, "values": ["B0B0bar_Btag-HD_Cocktail", "B0B0bar_Btag-SL_e_mu_tau_Bsig-HD_SL_Cocktail", "B0B0bar_generic", "B0B0bar_K0nunu", "B0B0bar_K0nunu_SL_e_mu_tau", "B0B0bar_Kstar0nunu_Kpi", "B0B0bar_Kstar0nunu_Kpi_SL_e_mu_tau", "B+B-_Btag-HD_Cocktail", "B+B-_Btag-SL_e_mu_tau_Bsig-HD_SL_Cocktail", "B+B-_generic", "B+B-_K+nunu", "B+B-_K+nunu_SL_e_mu_tau", "B+B-_Kstar+nunu", "B+B-_Kstar+nunu_SL_e_mu_tau", "B+B-_taunu_SL_e_mu_tau", "bhabha_bhwide", "ccbar", "tau+tau-_kk2f", "uds", "udsc", "Upsilon4S_generic"]},
{"name": "bkg_mixing", "label": "Background Mixing Type", "customValue": True, "values": ["All", "NoPair", "NoMixing"]},
{"name": "analysis_type", "label": "Analysis Type", "customValue": True, "values": ["BtoKNuNu", "BtoKstarNuNu", "HadRecoil", "SemiLepKplusNuNu"]}
]
for parameter in parameters:
asksParameter(parameter)
####################
# FULL Simulation session
####################
elif index == 1:
new_dataset['session'] = 'fullsim'
parameters = [
{"name": "evt_file", "label": "Events per file", "customValue": True, "values": []},
{"name": "sim_type", "label": "Simulation Type", "customValue": False, "values": ["fullsim", "background_frame"]},
{"name": "generator", "label": "Generator", "customValue": False, "values": ["RadBhaBha", "singleparticle"]},
{"name": "dg", "label": "Geometry", "customValue": True, "values": ["Geometry_CIPE", "Geometry_CIPE_BGO", "Geometry_CIPE_CSI", "Geometry_CIPE_V00-00-02"]},
{"name": "pl", "label": "Physics list", "customValue": True, "values": ["QGSP", "QGSP_BERT", "QGSP_BERT_HP"]},
{"name": "g4ver", "label": "Geant 4 version", "customValue": True, "values": ["9.2", "9.3"]},
{"name": "opt_photons", "label": "Optical Photons", "customValue": False, "values": ["OFF", "ON"]}
]
radbhabha = [
{"name": "brunobbbminde", "label": "Min. Delta E", "customValue": True, "values": []}
]
singleParticle = [
{"name": "brunopdg", "label": "PDG Code", "customValue": True, "values": []},
{"name": "brunothetamin", "label": "Theta min.", "customValue": True, "values": []},
{"name": "brunothetamax", "label": "Theta max.", "customValue": True, "values": []},
{"name": "brunophimin", "label": "Phi min.", "customValue": True, "values": []},
{"name": "brunophimax", "label": "Phi max.", "customValue": True, "values": []},
{"name": "brunoemin", "label": "Energy (GeV) min.", "customValue": True, "values": []},
{"name": "brunoemax", "label": "Energy (GeV) max.", "customValue": True, "values": []}
]
for parameter in parameters:
value = asksParameter(parameter)
# parameter dependencies management
if parameter['name'] == 'generator':
if value == 'singleparticle':
parameters.extend(singleParticle)
elif value == 'RadBhaBha':
parameters.extend(radbhabha)
####################
# ANALYSIS session
####################
elif index == 2:
new_dataset['session'] = 'analysis'
else:
raise GangaException('Invalid selection.')
while True:
free_string = raw_input('\nEnter free string: ')
max_length = 128
if len(free_string) <= max_length:
new_dataset['parameters']['free_string'] = free_string
break
else:
print('Free string must be <= %d char long.' % max_length)
# dataset-site relation set
new_dataset['site'] = getConfig('SuperB')['submission_site']
new_dataset['owner'] = utils.getOwner()
new_dataset['dataset_id'] = str(objectid.ObjectId())
print('\nNew dataset details:')
self.printDatasetDetail(new_dataset)
value = ''
while True:
value = raw_input('Type \'yes\' to confirm the dataset creation or (q)uit: ')
if value == 'yes':
break
elif value == 'q':
raise utils.QuitException()
sql = '''INSERT INTO analysis_dataset
(owner, dataset_id, session, parameters, status)
VALUES (%s, decode(%s, 'hex'), %s, %s, 'prepared');
INSERT INTO analysis_dataset_site
(dataset_id, site)
VALUES (decode(%s, 'hex'), %s);'''
params = (new_dataset['owner'],
new_dataset['dataset_id'],
new_dataset['session'],
new_dataset['parameters'],
new_dataset['dataset_id'],
new_dataset['site'])
db.write(sql, params)
def printDatasets(self, datasets):
''' Given the heterogeneous dataset list, the method splits it in
categories and build the table per session. A unique id all over
the sessions permit the user to select univocally a dataset. All
the metadata will be printed with the parent chain: a parent dataset
is defined as the one used as input to create the child one. The
method is public but not exported in GPI'''
# check the term width
(width, height) = utils.getTerminalSize()
if width < 160:
logger.error("Your terminal's width is %d; must be at least 160\nYou can make your font size smaller" % width)
raise utils.QuitException()
# better a named tuple but available in Python 2.6 only
# fullsim: the table elements should be common to all the dataset keys in the list
# print table rules:
grouped_datasets = [
# 0
{'title': 'Fastsim Official Production',
'dataset': list(),
'order_by': ['prod_series', 'analysis', 'generator', 'dg', 'bkg_mixing', 'analysis_type'],
'columns': ['id', 'prod_series', 'analysis', 'generator', 'dg', 'bkg_mixing', 'analysis_type', 'status']
},
# 1
{'title': 'Fastsim Personal Production',
'dataset': list(),
'order_by': ['free_string', 'analysis', 'generator', 'dg', 'bkg_mixing', 'analysis_type'],
'columns': ['id', 'free_string', 'analysis', 'generator', 'dg', 'bkg_mixing', 'analysis_type', 'status'],
},
# 2
{'title': 'Fullsim Official Production',
'dataset': list(),
'order_by': ['prod_series', 'simtype', 'generator', 'dg', 'pl', 'g4ver', 'opt_photons'],
'columns': ['id', 'prod_series', 'simtype', 'generator', 'dg', 'pl', 'g4ver', 'opt_photons', 'status']
},
# 3
{'title': 'Fullsim Personal Production',
'dataset': list(),
'order_by': ['free_string', 'generator', 'dg', 'pl', 'g4ver', 'opt_photons'],
'columns': ['id', 'free_string', 'generator', 'dg', 'pl', 'g4ver', 'opt_photons', 'status']
},
# 4
{'title': 'Analysis',
'dataset': list(),
'order_by': ['free_string', 'creation_date'],
'columns': ['id', 'free_string', 'creation_date', 'status']
}
]
for dataset in datasets:
# put sub dictionary elements to level zero dictionary
for key, value in dataset.items():
if type(dataset[key]) is types.DictType:
for key1, value1 in dataset[key].iteritems():
dataset[key1] = value1
#del dataset[key]
# dataset selection
if dataset['session'] == 'fastsim':
if dataset['owner'] == 'Official':
grouped_datasets[0]['dataset'].append(dataset)
else:
grouped_datasets[1]['dataset'].append(dataset)
elif dataset['session'] == 'fullsim':
if dataset['owner'] == 'Official':
grouped_datasets[2]['dataset'].append(dataset)
else:
grouped_datasets[3]['dataset'].append(dataset)
elif dataset['session'] == 'analysis':
grouped_datasets[4]['dataset'].append(dataset)
else:
raise GangaException('session not recognized: %s' % dataset['session'])
i = 0
# field sort, adding id and print
for group in grouped_datasets:
if len(group['dataset']) > 0:
print('\n%s' % group['title'])
# dictionary sorting
group['dataset'] = sorted(group['dataset'], key=lambda elem: ('%s ' * len(group['order_by'])) % tuple([elem[d] for d in group['order_by']]))
# id adding
for dataset in group['dataset']:
dataset['id'] = i
i += 1
print(utils.format_dict_table(group['dataset'], group['columns']))
# ask for input and print dataset details
if i == 1:
index = 0
print('\nAutomatically selected the only entry')
else:
print('\nChoose the dataset:')
index = utils.getIndex(maxExclusive=i)
# Object oriented solution to investigate and/or binary search
# datasets have been grouped per print rules
for group in grouped_datasets:
for d in group['dataset']:
if d['id'] == index:
for dataset in datasets:
if dataset['dataset_id'] == d['dataset_id']:
return dataset
def createDataset(self):
'''Interactive method to guide the user in dataset creation procedure.
If the dataset is a 'personal production' type, force user to provide
a filter key.'''
def asksParameter(parameter):
'''Interactive method requesting user the value of each parameter
per session (FastSim, FullSim, Analysis)'''
if parameter['customValue'] and len(parameter['values']) == 0:
value = raw_input('\nEnter %s: ' % parameter['label'])
elif not parameter['customValue'] and len(
parameter['values']) == 0:
raise GangaException(
'Invalid rule (customValue:False and values=0).')
else:
table = list()
i = 0
for value in parameter['values']:
table.append({'id': i, 'value': value})
i += 1
if parameter['customValue']:
table.append({'id': i, 'value': 'Enter a custom value'})
print('\nChoose %s:' % parameter['label'])
column_names = ('id', 'value')
print(utils.format_dict_table(table, column_names))
index = utils.getIndex(maxExclusive=len(table))
if parameter['customValue'] and index == len(table) - 1:
value = raw_input('Custom value: ')
else:
value = table[index]['value']
# parameter insertion in dictionary. It will be subsequently
#inserted into dataset analysis bookkeeping table, hstore field
new_dataset['parameters'][parameter['name']] = value
return value
type = [
dict(id=0, dataset_type='FastSim Personal Production'),
dict(id=1, dataset_type='FullSim Personal Production'),
dict(id=2, dataset_type='Analysis'),
]
column_names = ('id', 'dataset_type')
print(utils.format_dict_table(type, column_names))
index = utils.getIndex(maxExclusive=len(type))
new_dataset = dict()
new_dataset['parameters'] = dict()
####################
# FAST Simulation session
####################
# parameter check: mandatory, free string param management
# TODO: parameter type check, evaluate the config file option to store parameters
if index == 0:
new_dataset['session'] = 'fastsim'
parameters = [{
"name": "evt_file",
"label": "Events per file",
"customValue": True,
"values": []
}, {
"name":
"analysis",
"label":
"Analysis",
"customValue":
True,
"values": [
"BtoKNuNu", "BtoKstarNuNu", "DstD0ToXLL", "DstD0ToXLL",
"Generics", "HadRecoilCocktail", "KplusNuNu",
"SLRecoilCocktail", "tau->3mu"
]
}, {
"name": "dg",
"label": "Geometry",
"customValue": True,
"values": ["DG_4", "DG_4a", "DG_BaBar"]
}, {
"name":
"generator",
"label":
"Generator",
"customValue":
True,
"values": [
"B0B0bar_Btag-HD_Cocktail",
"B0B0bar_Btag-SL_e_mu_tau_Bsig-HD_SL_Cocktail",
"B0B0bar_generic", "B0B0bar_K0nunu",
"B0B0bar_K0nunu_SL_e_mu_tau", "B0B0bar_Kstar0nunu_Kpi",
"B0B0bar_Kstar0nunu_Kpi_SL_e_mu_tau",
"B+B-_Btag-HD_Cocktail",
"B+B-_Btag-SL_e_mu_tau_Bsig-HD_SL_Cocktail",
"B+B-_generic", "B+B-_K+nunu", "B+B-_K+nunu_SL_e_mu_tau",
"B+B-_Kstar+nunu", "B+B-_Kstar+nunu_SL_e_mu_tau",
"B+B-_taunu_SL_e_mu_tau", "bhabha_bhwide", "ccbar",
"tau+tau-_kk2f", "uds", "udsc", "Upsilon4S_generic"
]
}, {
"name": "bkg_mixing",
"label": "Background Mixing Type",
"customValue": True,
"values": ["All", "NoPair", "NoMixing"]
}, {
"name":
"analysis_type",
"label":
"Analysis Type",
"customValue":
True,
"values":
["BtoKNuNu", "BtoKstarNuNu", "HadRecoil", "SemiLepKplusNuNu"]
}]
for parameter in parameters:
asksParameter(parameter)
####################
# FULL Simulation session
####################
elif index == 1:
new_dataset['session'] = 'fullsim'
parameters = [{
"name": "evt_file",
"label": "Events per file",
"customValue": True,
"values": []
}, {
"name": "sim_type",
"label": "Simulation Type",
"customValue": False,
"values": ["fullsim", "background_frame"]
}, {
"name": "generator",
"label": "Generator",
"customValue": False,
"values": ["RadBhaBha", "singleparticle"]
}, {
"name":
"dg",
"label":
"Geometry",
"customValue":
True,
"values": [
"Geometry_CIPE", "Geometry_CIPE_BGO", "Geometry_CIPE_CSI",
"Geometry_CIPE_V00-00-02"
]
}, {
"name": "pl",
"label": "Physics list",
"customValue": True,
"values": ["QGSP", "QGSP_BERT", "QGSP_BERT_HP"]
}, {
"name": "g4ver",
"label": "Geant 4 version",
"customValue": True,
"values": ["9.2", "9.3"]
}, {
"name": "opt_photons",
"label": "Optical Photons",
"customValue": False,
"values": ["OFF", "ON"]
}]
radbhabha = [{
"name": "brunobbbminde",
"label": "Min. Delta E",
"customValue": True,
"values": []
}]
singleParticle = [{
"name": "brunopdg",
"label": "PDG Code",
"customValue": True,
"values": []
}, {
"name": "brunothetamin",
"label": "Theta min.",
"customValue": True,
"values": []
}, {
"name": "brunothetamax",
"label": "Theta max.",
"customValue": True,
"values": []
}, {
"name": "brunophimin",
"label": "Phi min.",
"customValue": True,
"values": []
}, {
"name": "brunophimax",
"label": "Phi max.",
"customValue": True,
"values": []
}, {
"name": "brunoemin",
"label": "Energy (GeV) min.",
"customValue": True,
"values": []
}, {
"name": "brunoemax",
"label": "Energy (GeV) max.",
"customValue": True,
"values": []
}]
for parameter in parameters:
value = asksParameter(parameter)
# parameter dependencies management
if parameter['name'] == 'generator':
if value == 'singleparticle':
parameters.extend(singleParticle)
elif value == 'RadBhaBha':
parameters.extend(radbhabha)
####################
# ANALYSIS session
####################
elif index == 2:
new_dataset['session'] = 'analysis'
else:
raise GangaException('Invalid selection.')
while True:
free_string = raw_input('\nEnter free string: ')
max_length = 128
if len(free_string) <= max_length:
new_dataset['parameters']['free_string'] = free_string
break
else:
print('Free string must be <= %d char long.' % max_length)
# dataset-site relation set
new_dataset['site'] = getConfig('SuperB')['submission_site']
new_dataset['owner'] = utils.getOwner()
new_dataset['dataset_id'] = str(objectid.ObjectId())
print('\nNew dataset details:')
self.printDatasetDetail(new_dataset)
value = ''
while True:
value = raw_input(
'Type \'yes\' to confirm the dataset creation or (q)uit: ')
if value == 'yes':
break
elif value == 'q':
raise utils.QuitException()
sql = '''INSERT INTO analysis_dataset
(owner, dataset_id, session, parameters, status)
VALUES (%s, decode(%s, 'hex'), %s, %s, 'prepared');
INSERT INTO analysis_dataset_site
(dataset_id, site)
VALUES (decode(%s, 'hex'), %s);'''
params = (new_dataset['owner'], new_dataset['dataset_id'],
new_dataset['session'], new_dataset['parameters'],
new_dataset['dataset_id'], new_dataset['site'])
db.write(sql, params)
