def main():
parse_args(config)
set_logger(config, logger)
if config['daemon']:
daemon()
pid_file = config['pidfile']
mk_pid_file(pid_file)
run_server(config)
# Load the parameters from json file
args = parser.parse_args()
json_path = os.path.join(args.model_dir, 'params.json')
assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
params = utils.Params(json_path)
# use GPU if available
params.cuda = torch.cuda.is_available()
# Set the random seed for reproducible experiments
random.seed(230)
torch.manual_seed(230)
if params.cuda: torch.cuda.manual_seed(230)
# Set the logger
utils.set_logger(os.path.join(args.model_dir, 'train.log'))
# Create the input data pipeline
logging.info("Loading the datasets...")
# fetch dataloaders, considering full-set vs. sub-set scenarios
if params.subset_percent < 1.0:
train_dl = data_loader.fetch_subset_dataloader('train', params)
else:
train_dl = data_loader.fetch_dataloader('train', params)
dev_dl = data_loader.fetch_dataloader('dev', params)
logging.info("- done.")
"""Based on the model_version, determine model/optimizer and KD training mode
def main():
# argument processing
parser = argparse.ArgumentParser(description='Detect your PHP dead code')
parser.add_argument('--dir', dest='dir', metavar='dir_path',
type=str, nargs=1, default=None,
help='Make ZomPHP process that directory')
parser.add_argument('--ignore-sub-dirs', dest='ignore_sub_dirs',
metavar='dir_path', type=str, nargs='+', default=[],
help='A directory path (or list of those) that won\'t '
'be processed (only makes sense when used with the '
'--dir option)')
parser.add_argument('--files', dest='files', metavar='file_path',
type=str, nargs='+', default=[],
help='A file or list of files (given as absolute paths'
') to have processed by ZomPHP')
parser.add_argument('--strict', dest='strict', action='store_const',
const=True, default=False, help='If set to true, will'
' guarantee that any function NOT marked is indeed '
'used, but might also yield more false negatives (this'
' option should only be used if you have files '
'containing functions with the same name)')
parser.add_argument('--path-translation', dest='path_translation',
metavar='local_path path_in_db', type=str, nargs='+',
default=[], help='A list of couples of paths to '
'translate (useful if running the code in a different '
'location than the one running the PHP code)')
parser.add_argument('--logging-level', dest='logging_level', metavar='level',
type=str, nargs=1, default=None, help='A logging '
'level to override the one set in the settings file')
args = parser.parse_args()
# start the logger
utils.set_logger()
# some sanity checks
def check_abs_path(path, option_name):
# helper function, checks the paths are absolute, and translates them to real paths
if not path:
return path
if isinstance(path, (tuple, list)):
return [check_abs_path(p, option_name) for p in path]
if os.path.isabs(path):
return os.path.realpath(path)
logging.error('The --%s option requires using absolute paths (you entered %s) exiting' % (option_name, path))
sys.exit(1)
if bool(args.dir) == bool(args.files):
logging.error('You must specify exactly one of the --dir or --files options, exiting')
sys.exit(1)
args.dir = check_abs_path(args.dir, 'dir')
args.files = check_abs_path(args.files, 'files')
if args.ignore_sub_dirs:
if args.dir:
args.ignore_sub_dirs = check_abs_path(args.ignore_sub_dirs, 'ignore-sub-dirs')
else:
logging.warning('Ignoring the --ignore-sub-dirs option, that option can only be used together with the --dir option')
translator = utils.PathTranslator.build_translator(args.path_translation)
# down to work!
bckend = backend.get_new_backend()
if args.dir:
bckend.process_directory(args.dir[0], strict=args.strict, ignore_sub_dirs=args.ignore_sub_dirs, translator=translator)
else:
# then it must be --files
for fle in args.files:
bckend.process_file(fle, args.strict, translator=translator)
logging.info(bckend.stats)
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.cross_validation import train_test_split
from sklearn import linear_model, datasets
from sklearn import metrics
from sklearn import preprocessing
from sklearn.cross_validation import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from pandas import *
from settings import DATA_DIR, LOG_DIR
import utils
TOTAL_INDEX = 'djia gspc ixic'.split() # dow jones. snp500, nasdaq, vol
EXPID = utils.get_expid()
utils.set_logger('%s/%s.log' % (LOG_DIR, EXPID), 'DEBUG')
# TODO: log configurations (ex: parsing method etc) and/or commit id
def openfiles(filename, arg):
data = pd.read_csv(filename, sep='\t', header = 0)
data = data.where((pd.notnull(data)), '') # Replace np.nan with ''
if arg == 100: # X
columns = ['id', 'text', 'closePrice', 'week', 'month', 'quater', 'year','djia', 'gspc', 'ixic', 'vix']
data.columns = columns
value = pd.DataFrame(data)
value.index = data['id']
else: # y
columns = TOTAL_INDEX[arg]
value = pd.DataFrame(data[TOTAL_INDEX[arg]])
"""
# Load the parameters
args = parser.parse_args()
json_path = os.path.join(args.model_dir, 'params.json')
assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
params = utils.Params(json_path)
# use GPU if available
params.cuda = torch.cuda.is_available() # use GPU is available
# Set the random seed for reproducible experiments
torch.manual_seed(230)
if params.cuda: torch.cuda.manual_seed(230)
# Get the logger
utils.set_logger(os.path.join(args.model_dir, 'analysis.log'))
# Create the input data pipeline
logging.info("Loading the dataset...")
# fetch dataloaders
# train_dl = data_loader.fetch_dataloader('train', params)
# dev_dl = data_loader.fetch_dataloader('dev', params)
dataloader = data_loader.fetch_dataloader(args.dataset, params)
logging.info("- done.")
# Define the model graph
model = resnet.ResNet18().cuda() if params.cuda else resnet.ResNet18()
# fetch loss function and metrics
# Launch training with this config
cmd = "{python} train.py --model_dir={model_dir}".format(python=PYTHON,
model_dir=model_dir)
print(cmd)
check_call(cmd, shell=True)
if __name__ == "__main__":
# Load the "reference" parameters from parent_dir json file
args = parser.parse_args()
json_path = os.path.join(args.parent_dir, 'params.json')
assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
params = utils.Params(json_path)
# Set the logger
utils.set_logger(os.path.join(args.parent_dir, 'search_hyperparameters.log'))
'''
Temperature and alpha search for KD on CNN (teacher model picked in params.json)
Perform hypersearch (empirical grid): distilling 'temperature', loss weight 'alpha'
'''
# hyperparameters for KD
alphas = [0.99, 0.95, 0.5, 0.1, 0.05]
temperatures = [20., 10., 8., 6., 4.5, 3., 2., 1.5]
logging.info("Searching hyperparameters...")
logging.info("alphas: {}".format(alphas))
logging.info("temperatures: {}".format(temperatures))
for alpha in alphas:
