def main(args):
sys.argv = args
# Parse arguments
parser = ArgumentParser()
parser.add_argument('-local_step_dbs', required=True, help='Path to db holding local step results.')
args, unknown = parser.parse_known_args()
local_dbs = path.abspath(args.local_step_dbs)
# Load local nodes output
args_X, args_Y,CategoricalVariablesWithDistinctValues, GlobalHist = multipleHist2_Loc2Glob_TD.load(local_dbs).get_data()
init_logger()
logging.info(["GlobalHist= ", GlobalHist])
# Histogram modification due to privacy --> Move it to local.py
for key in GlobalHist:
for i in xrange(len(GlobalHist[key]['Data'])):
if isinstance(GlobalHist[key]['Data'][i], list):
for j in xrange(len(GlobalHist[key]['Data'][i])):
if GlobalHist[key]['Data'][i][j] <= PRIVACY_MAGIC_NUMBER:
GlobalHist[key]['Data'][i][j] = 0
else:
if GlobalHist[key]['Data'][i] <= PRIVACY_MAGIC_NUMBER:
GlobalHist[key]['Data'][i] = 0
# Return the algorithm's output
#raise ValueError (args_X, args_Y,CategoricalVariablesWithDistinctValues,GlobalHist)
global_out = histogramToHighchart(GlobalHist, args_X, args_Y)
set_algorithms_output_data(global_out)
def main(args):
sys.argv = args
init_logger()
# Parse arguments
parser = ArgumentParser()
parser.add_argument(
'-cur_state_pkl',
required=True,
help='Path to the pickle file holding the current state.')
parser.add_argument(
'-prev_state_pkl',
required=True,
help='Path to the pickle file holding the previous state.')
parser.add_argument('-local_step_dbs',
required=True,
help='Path to db holding local step results.')
args, unknown = parser.parse_known_args()
fname_cur_state = path.abspath(args.cur_state_pkl)
fname_prev_state = path.abspath(args.prev_state_pkl)
local_dbs = path.abspath(args.local_step_dbs)
# Load global state
global_state = StateData.load(fname_prev_state).data
globalTreeJ = global_state['globalTree'].tree_to_json()
myresult = {"result": [{"type": "application/json", "data": globalTreeJ}]}
t1 = global_state['t1']
t2 = time.localtime(time.time())
t0 = ['yy', 'mm', 'dd', 'hh', 'min', 'sec', 'wday', 'yday', 'isdst']
logging.info(" Time: ")
for i in range(len(t1)):
logging.info([t0[i], t2[i], t1[i], t2[i] - t1[i]])
set_algorithms_output_data(myresult)
def main(args):
sys.argv = args
# Parse arguments
parser = ArgumentParser()
parser.add_argument(
'-prev_state_pkl',
required=True,
help='Path to the pickle file holding the previous state.')
parser.add_argument('-global_step_db',
required=True,
help='Path to db holding global step results.')
args, unknown = parser.parse_known_args()
fname_prev_state = path.abspath(args.prev_state_pkl)
global_db = path.abspath(args.global_step_db)
# Load local state
local_state = StateData.load(fname_prev_state).get_data()
# Load global node output
globalStatistics = Global2Local_TD.load(global_db).get_data()['global_in']
init_logger()
logging.info([
"args_X, args_Y, args_bins, dataSchema, CategoricalVariablesWithDistinctValues:",
local_state['args_X'],
local_state['args_Y'],
local_state['args_bins'],
local_state['dataSchema'],
local_state['CategoricalVariablesWithDistinctValues'],
])
logging.debug(["dataFrame=", local_state['dataFrame'][0:5]])
logging.info(["globalStatistics=", globalStatistics])
# Run algorithm local step
Hist = run_local_step(
local_state['args_X'], local_state['args_Y'], local_state['args_bins'],
local_state['dataSchema'],
local_state['CategoricalVariablesWithDistinctValues'],
local_state['dataFrame'], globalStatistics)
# Pack results
local_out = multipleHist2_Loc2Glob_TD(
local_state['args_X'], local_state['args_Y'],
local_state['CategoricalVariablesWithDistinctValues'], Hist)
# Return the output data
local_out.transfer()
def main(args):
# Parse arguments
sys.argv = args
parser = ArgumentParser()
parser.add_argument('-local_step_dbs',
required=True,
help='Path to local db.')
args, unknown = parser.parse_known_args()
local_dbs = path.abspath(args.local_step_dbs)
#Get global data
globalStatistics = multipleHist1_Loc2Glob_TD.load(local_dbs).get_data()
init_logger()
logging.info(["globalStatistics:", globalStatistics])
#raise ValueError(globalStatistics)
global_out = Global2Local_TD(global_in=globalStatistics)
# Return the algorithm's output
global_out.transfer()
def main(args):
sys.argv = args
# Parse arguments
parser = ArgumentParser()
parser.add_argument('-x',
required=True,
help='Variable names, comma seperated ')
parser.add_argument('-y',
required=True,
help='Categorical variables names, comma seperated.')
parser.add_argument(
'-bins',
required=True,
help='Dictionary of variables names (key) and number of bins (value)')
parser.add_argument('-input_local_DB',
required=True,
help='Path to local db.')
parser.add_argument('-db_query',
required=True,
help='Query to be executed on local db.')
parser.add_argument(
'-cur_state_pkl',
required=True,
help='Path to the pickle file holding the current state.')
args, unknown = parser.parse_known_args()
query = args.db_query
fname_cur_state = path.abspath(args.cur_state_pkl)
fname_loc_db = path.abspath(args.input_local_DB)
#if args.x == '':
# raise ExaremeError('Field x must be non empty.')
# Get data
if args.x == '':
args_X = list(args.y.replace(' ', '').split(','))
args_Y = []
varNames = "'" + "','".join(list(args.y.replace(' ',
'').split(','))) + "'"
else:
args_X = list(args.y.replace(' ', '').split(','))
args_Y = list(args.x.replace(' ', '').split(','))
varNames = "'" + "','".join(list(args.y.replace(
' ', '').split(','))) + "','" + "','".join(
list(args.x.replace(' ', '').split(','))) + "'"
if args.bins == '':
args_bins = {}
else:
args_bins = json.loads(args.bins)
#args_bins = dict( (str(key), val) for key, val in args_bins.items())
queryMetadata = "select * from metadata where code in (" + varNames + ");"
dataSchema, metadataSchema, metadata, dataFrame = query_database(
fname_db=fname_loc_db, queryData=query, queryMetadata=queryMetadata)
CategoricalVariablesWithDistinctValues = variable_categorical_getDistinctValues(
metadata)
#Checking bins input
for varx in args_X:
if varx not in CategoricalVariablesWithDistinctValues:
if varx not in args_bins:
raise ExaremeError(
'Bin value is not defined for one at least non-categorical variable. i.e. '
+ varx)
# Run algorithm local step
localStatistics = run_local_step(args_X, args_Y, args_bins, dataSchema,
CategoricalVariablesWithDistinctValues,
dataFrame)
# Save local state
local_state = StateData(args_X=args_X,
args_Y=args_Y,
args_bins=args_bins,
dataSchema=dataSchema,
CategoricalVariablesWithDistinctValues=
CategoricalVariablesWithDistinctValues,
dataFrame=dataFrame)
local_state.save(fname=fname_cur_state)
init_logger()
logging.info([
"args_X, args_Y, args_bins, dataSchema, CategoricalVariablesWithDistinctValues:",
args_X, args_Y, args_bins, dataSchema,
CategoricalVariablesWithDistinctValues
])
logging.debug(["dataFrame:", dataFrame[0:5]])
logging.info(["localStatistics:", localStatistics])
# Transfer local output
local_out = multipleHist1_Loc2Glob_TD(localStatistics)
#raise ValueError( local_out.get_data())
local_out.transfer()