def main(args):
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
files = get_files(args.folder)
for file_path in files:
file = open(str(file_path), "r")
raw_content = file.read()
file.close()
result_file_path = os.path.join(args.output,
os.path.basename(file_path))
result_file = open(str(result_file_path), "w")
sentences = [
s for s in raw_content.decode('utf-8').strip().split("\n")
if len(s) > 0 and s != "\n"
]
for sentence in sentences:
if sentence == graphseg_delimeter:
result_file.write(wiki_delimiter.encode('utf-8'))
else:
result_file.write(sentence.encode('utf-8'))
result_file.write("\n".encode('utf-8'))
def main(args):
sys.path.append(str(Path(__file__).parent))
checkpoint_path = Path(args.checkpoint_dir)
checkpoint_path.mkdir(exist_ok=True)
logger = utils.setup_logger(__name__, os.path.join(args.checkpoint_dir, 'train.log'))
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
# logger.debug('Running with config %s', utils.config)
# configure(os.path.join('runs', args.expname))
word2vec = None
if not args.infer:
dataset_path = utils.config['choidataset']
train_dataset = ChoiDataset(dataset_path, word2vec)
dev_dataset = ChoiDataset(dataset_path, word2vec)
test_dataset = ChoiDataset(dataset_path, word2vec)
train_dl = DataLoader(train_dataset, batch_size=args.bs, collate_fn=collate_fn, shuffle=True,
num_workers=args.num_workers)
dev_dl = DataLoader(dev_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
test_dl = DataLoader(test_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
assert bool(args.model) ^ bool(args.load_from) # exactly one of them must be set
if args.model:
model = import_model(args.model)
elif args.load_from:
with open(args.load_from, 'rb') as f:
model = torch.load(f)
model.train()
model = maybe_cuda(model)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
if not args.infer:
best_val_pk = 1.0
for j in range(args.epochs):
train(model, args, j, train_dl, logger, optimizer)
with (checkpoint_path / 'model{:03d}.t7'.format(j)).open('wb') as f:
torch.save(model, f)
val_pk, threshold = validate(model, args, j, dev_dl, logger)
if val_pk < best_val_pk:
test_pk = test(model, args, j, test_dl, logger, threshold)
logger.debug(
colored(
'Current best model from epoch {} with p_k {} and threshold {}'.format(j, test_pk, threshold),
'green'))
best_val_pk = val_pk
with (checkpoint_path / 'Meilleur_model.t7'.format(j)).open('wb') as f:
torch.save(model, f)
def main(args):
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
if not args.test:
word2vec = gensim.models.KeyedVectors.load_word2vec_format(
utils.config['word2vecfile'], binary=True)
else:
word2vec = None
with open(args.model, 'rb') as f:
model = torch.load(f)
model.eval()
segment(args.path, model, word2vec, args.output, wiki=args.wiki)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.config = read_config_file()
self.db = model.get_db(self.config['db_path'])
self.books = []
for book in self.db.query(model.Book).all():
self.books.append(book)
builder = Gtk.Builder()
builder.add_from_file('./gui/minerva.glade')
self.window = builder.get_object('minerva_main')
self.vbox = builder.get_object('vbox')
self.tv_filters = builder.get_object('tv_filters')
self.statusbar = builder.get_object('statusbar')
self.btn_edit = builder.get_object('btn_edit')
self.btn_delete = builder.get_object('btn_delete')
self.books = BookList(parent=self, books=self.books)
self.filters = Gtk.ListStore(str)
self.search_entry = Gtk.SearchEntry()
setup_info_bar(self)
self.vbox.pack_end(self.info_bar, False, True, 0)
self._setup_filters(self.filters, self.tv_filters)
builder.get_object('hpaned').pack2(self.books)
builder.get_object('hbox').pack_start(self.search_entry, True, True, 0)
builder.connect_signals(self)
self.search_entry.connect('search-changed', self.on_search_changed)
self.window.connect('key-press-event', self.on_key_press_event)
def __call__(self, parser, namespace, value, option_string=None):
config = read_config_file()
format = namespace.format if namespace.format else config.get('DEFAULT','format')
oauth = config.getboolean('DEFAULT','oauth')
# Checking ig OAuth params are defined
if oauth :
oauth = YOAuth(None, None, from_file=config.get('auth','from_file'))
attr = {
'community': True,
'format': format,
#'jsonCompact': namespace.jsonCompact if namespace.jsonCompact else config.getboolean(format, 'jsonCompact'),
'debug': namespace.debug if namespace.debug else config.getboolean(format, 'debug'),
'oauth': oauth
}
yql = MYQL(**attr)
yql.diagnostics = namespace.diagnostics if namespace.diagnostics else config.getboolean(format, 'diagnostics')
for v in value:
response = yql.rawQuery(v)
if not response.status_code == 200:
print(response.content)
sys.exit(1)
if format == 'json':
print(pretty_json(response.content))
else:
print(pretty_xml(response.content))
sys.exit(0)
def main():
print "Cancelling orders"
print "-----------------"
global config_dict
inst_token = []
config_dict = utils.read_config_file()
acccess_toke, request_token, kite = kite_utils.login_kite(None)
print kite
orders = kite.orders()
print "======================================================="
for each in orders:
if each['parent_order_id'] != None:
print each
kite.cancel_order(kite.VARIETY_BO,
each['order_id'],
parent_order_id=each['parent_order_id'])
else:
kite.cancel_order(kite.VARIETY_BO,
each['order_id'],
parent_order_id=None)
print "======================================================="
def main(args):
sys.path.append(str(Path(__file__).parent))
logger = utils.setup_logger(__name__, 'cross_validate_choi.log')
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
logger.debug('Running with config %s', utils.config)
configure(os.path.join('runs', args.expname))
if not args.test:
word2vec = gensim.models.KeyedVectors.load_word2vec_format(utils.config['word2vecfile'], binary=True)
else:
word2vec = None
dataset_path = Path(args.flat_choi)
with open(args.load_from, 'rb') as f:
model = torch.load(f)
model.eval()
model = maybe_cuda(model)
test_accuracy = accuracy.Accuracy()
for j in range(5):
validate_folder_numbers = range(5)
validate_folder_numbers.remove(j)
validate_folder_names = [dataset_path.joinpath(str(num)) for num in validate_folder_numbers]
dev_dataset = ChoiDataset(dataset_path , word2vec, folder=True, folders_paths=validate_folder_names)
test_dataset = ChoiDataset(dataset_path, word2vec, folder=True, folders_paths=[dataset_path.joinpath(str(j))])
dev_dl = DataLoader(dev_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
test_dl = DataLoader(test_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
_, threshold = validate(model, args, j, dev_dl, logger)
test_pk = test(model, args, j, test_dl, logger, threshold, test_accuracy)
logger.debug(colored('Cross validation section {} with p_k {} and threshold {}'.format(j, test_pk, threshold),'green'))
cross_validation_pk, _ = test_accuracy.calc_accuracy()
print ('Final cross validaiton Pk is: ' + str(cross_validation_pk))
logger.debug(
colored('Final cross validaiton Pk is: {}'.format(cross_validation_pk), 'green'))
def main(dataset):
file_path = './config/' + dataset + '/AENet.cfg'
config = read_config_file(file_path)
create_dir(config['ckpt_dir'])
print('Test AE-Net | Dataset: {}'.format(dataset))
print('\nLoading configuration file {}...done'.format(file_path))
test_aenet_model(config)
def main(dataset):
file_path = './config/' + dataset + '/ACRegNet.cfg'
config = read_config_file(file_path)
create_dir(config['ckpt_dir'])
print('Training AC-Regnet | Dataset: {}'.format(dataset))
print('\nLoading configuration file {}...done'.format(file_path))
train_acregnet_model(config)
def make_prediction(input_data):
# load pipeline and make predictions
config = ut.read_config_file('config.yaml')
_titanic_pipe = joblib.load(
filename=config[0]['Paths'].get('output_model_path'))
# return predictions
results = _titanic_pipe.predict(input_data)
return results
def list_deployed_models_api() -> str:
api_token = connexion.request.form["api_token"]
log("[INFO] Calling list_models_api", api_token)
if auth_token(api_token):
models_config_list = utils.read_config_file()
log("[INFO] List of deployed models:\n" + str(models_config_list), api_token)
return str(models_config_list)
else:
return log("[INFO] Invalid token", api_token)
def main(args):
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
with Path(args.file).open('r') as f:
file_names = f.read().strip().split('\n')
word2vec = None
with open(args.model, 'rb') as f:
#model = torch.load(f)
#for run in cpu
model = torch.load(f, map_location='cpu')
model.eval()
for name in file_names:
if name:
segment(Path(name), model, word2vec, args.output)
def main(args):
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
algo_delimeter = graphseg_delimeter
files = get_files(args.folder)
acc = accuracy.Accuracy()
for file_path in files:
file = open(str(file_path), "r")
raw_content = file.read()
file.close()
sentences = [
s for s in raw_content.decode('utf-8').strip().split("\n")
if len(s) > 0 and s != "\n"
]
sentences_length = []
h = []
t = []
is_first_sentence = True
for sentence in sentences:
if sentence == truth:
if not is_first_sentence:
t[-1] = 1
continue
if sentence == algo_delimeter:
if not is_first_sentence:
h[-1] = 1
continue
words = extract_sentence_words(sentence)
sentences_length.append(len(words))
t.append(0)
h.append(0)
is_first_sentence = False
t[-1] = 1 # end of last segment
h[-1] = 1 # they already segment it correctly.
acc.update(h, t)
calculated_pk, calculated_windiff = acc.calc_accuracy()
print 'Pk: {:.4}.'.format(calculated_pk)
print 'Win_diff: {:.4}.'.format(calculated_windiff)
def main(args):
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
with Path(args.file).open('r') as f:
file_names = f.read().strip().split('\n')
if not args.test:
word2vec = gensim.models.KeyedVectors.load_word2vec_format(utils.config['word2vecfile'], binary=True)
else:
word2vec = None
with open(args.model, 'rb') as f:
model = torch.load(f)
model.eval()
for name in file_names:
if name:
segment(Path(name), model, word2vec, args.output, wiki=args.wiki)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', dest='config_file',
default='./settings')
parser.add_argument('-i', '--index', dest='xen_index',
default=XEN_INDEX)
args = parser.parse_args()
config = read_config_file(args.config_file)
cursor = connect_to_mysql(**config['mysql'])
dfs = load_dataframes(cursor)
data = calculate(dfs)
es_conn = connect_to_elasticsearch(**config['elasticsearch'])
write_to_elasticsearch(es_conn, data, args.xen_index)
def login_kite(request_token):
global config_dict
# read config file
config_dict = utils.read_config_file()
#kite = kite_utils.kite_login(request_token)
config = ConfigParser.ConfigParser()
config.read(config_dict['data_access'])
access_token = config.get('MAIN', 'DATA_ACCESS_TOKEN')
my_api = str(config_dict['kite_api_key'])
my_api_secret = str(config_dict['kite_api_secret'])
kite = KiteConnect(api_key=my_api)
url = kite.login_url()
print url
# Redirect the user to the login url obtained
# from kite.login_url(), and receive the request_token
# from the registered redirect url after the login flow.
# Once you have the request_token, obtain the access_token
# as follows.
# sys.argv[1] is access token that we get from login
if request_token == None:
kite.set_access_token(access_token)
else:
data = kite.generate_session(request_token, api_secret=my_api_secret)
kite.set_access_token(data["access_token"])
access_token = data["access_token"]
config.set('MAIN', 'DATA_ACCESS_TOKEN', data["access_token"])
with open(config_dict['data_access'], 'wb') as configfile:
config.write(configfile)
return my_api, access_token, kite
def run_training():
"""Train the model."""
# read training data
config = ut.read_config_file('config.yaml')
path = config[0]['Paths'].get('directory')
filename = config[0]['Paths'].get('data_filename')
extension = config[0]['Paths'].get('data_extension')
cols = config[2]['Feature_Groups'].get('data_columns')
target = config[2]['Feature_Groups'].get('target')
data = ut.load_data(path=path,
filename=filename,
extension=extension,
cols=cols)
# divide train and test
data[target] = data[target].astype(int)
X_train, X_test, y_train, y_test = train_test_split(data.drop(target,
axis=1),
data[target],
test_size=0.2,
random_state=0)
# fit pipeline
titanic_pipe.fit(X_train, y_train)
# save pipeline
joblib.dump(titanic_pipe, config[0]['Paths'].get('output_model_path'))
def main(args):
sys.path.append(str(Path(__file__).parent))
checkpoint_path = Path(args.checkpoint_dir)
checkpoint_path.mkdir(exist_ok=True)
logger = utils.setup_logger(__name__, os.path.join(args.checkpoint_dir, 'train.log'))
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
logger.debug('Running with config %s', utils.config)
configure(os.path.join('runs', args.expname))
if not args.test:
word2vec = gensim.models.KeyedVectors.load_word2vec_format(utils.config['word2vecfile'], binary=True)
else:
word2vec = None
if not args.infer:
if args.wiki:
dataset_path = Path(utils.config['wikidataset'])
train_dataset = WikipediaDataSet(dataset_path / 'train', word2vec=word2vec,
high_granularity=args.high_granularity)
dev_dataset = WikipediaDataSet(dataset_path / 'dev', word2vec=word2vec, high_granularity=args.high_granularity)
test_dataset = WikipediaDataSet(dataset_path / 'test', word2vec=word2vec,
high_granularity=args.high_granularity)
else:
dataset_path = Path(utils.config['choidataset'])
train_dataset = ChoiDataset(dataset_path / 'train', word2vec)
dev_dataset = ChoiDataset(dataset_path / 'dev', word2vec)
test_dataset = ChoiDataset(dataset_path / 'test', word2vec)
train_dl = DataLoader(train_dataset, batch_size=args.bs, collate_fn=collate_fn, shuffle=True,
num_workers=args.num_workers)
dev_dl = DataLoader(dev_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
test_dl = DataLoader(test_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
assert bool(args.model) ^ bool(args.load_from) # exactly one of them must be set
if args.model:
model = import_model(args.model)
elif args.load_from:
with open(args.load_from, 'rb') as f:
model = torch.load(f)
model.train()
model = maybe_cuda(model)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
if not args.infer:
best_val_pk = 1.0
for j in range(args.epochs):
train(model, args, j, train_dl, logger, optimizer)
with (checkpoint_path / 'model{:03d}.t7'.format(j)).open('wb') as f:
torch.save(model, f)
val_pk, threshold = validate(model, args, j, dev_dl, logger)
test_pk = test(model, args, j, test_dl, logger, threshold)
logger.debug(
colored(
'Current best model from epoch {} with p_k {} and threshold {}'.format(j, test_pk, threshold),
'green'))
best_val_pk = val_pk
else:
test_dataset = WikipediaDataSet(args.infer, word2vec=word2vec,
high_granularity=args.high_granularity)
test_dl = DataLoader(test_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
print(test(model, args, 0, test_dl, logger, 0.4))
import json
import orion
import cygnus
from utils import read_config_file
config = read_config_file('config.ini')
with open(config.get('device_schema_path')) as json_file:
data = json.load(json_file)
orion.init(config.get('orion_host'), config.get('orion_port'))
orion.register_entity(data, config.get('device_type'), config.get('device_id'), '0.0.0.0:4000')
orion.get_entities_by_type(config.get('device_type'))
#orion.get_entities_by_id(config.get('device_id'))
#orion.update_context(config.get('device_id'),
# config.get('device_type'), "true")
#orion.get_entities_by_id(config.get('device_id'))
#cygnus.init(config.get('cygnus_host'), config.get('cygnus_port'))
#cygnus.subscribe_attributes_change(config.get('device_type'), config.get('device_id'), ["status"], config.get('')
def main(args):
sys.path.append(str(Path(__file__).parent))
checkpoint_path = Path(args.checkpoint_dir)
checkpoint_path.mkdir(exist_ok=True)
logger = utils.setup_logger(__name__, os.path.join(args.checkpoint_dir, 'train.log'))
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
logger.debug('Running with config %s', utils.config)
configure(os.path.join('runs', args.expname))
if not args.test:
word2vec = gensim.models.KeyedVectors.load_word2vec_format(utils.config['word2vecfile'], binary=True)
else:
word2vec = None
# read the pretrained BERT sentence embeddings...
bound1 = 14900; bound2 = 2135; bound3 = 50; bound4 = 100; bound5 = 117; bound6 = 227;
bert_vec = []
bert_vec = read_sent_bert(bert_vec, '/ubc/cs/research/nlp/Linzi/seg/bert/bert_emb_train_cleaned.txt')
bert_vec = read_sent_bert(bert_vec, '/ubc/cs/research/nlp/Linzi/seg/bert/bert_emb_dev_cleaned.txt')
bert_vec = read_sent_bert(bert_vec, '/ubc/cs/research/nlp/Linzi/seg/bert/test1_data_emb_cleaned.txt')
bert_vec = read_sent_bert(bert_vec, '/ubc/cs/research/nlp/Linzi/seg/bert/bert_emb_test_2.txt')
bert_vec = read_sent_bert(bert_vec, '/ubc/cs/research/nlp/Linzi/seg/bert/bert_emb_test_3.txt')
bert_vec = read_sent_bert(bert_vec, '/ubc/cs/research/nlp/Linzi/seg/bert/bert_emb_test_4.txt')
bert_vec = read_sent_bert(bert_vec, '/ubc/cs/research/nlp/Linzi/seg/bert/bert_emb_test_wiki.txt')
train_bert = bert_vec[0:bound1]
dev_bert = bert_vec[bound1:(bound1+bound2)]
test_bert = bert_vec[(bound1+bound2):(bound1+bound2+bound3)]
test2_bert = bert_vec[(bound1+bound2+bound3):(bound1+bound2+bound3+bound4)]
test3_bert = bert_vec[(bound1+bound2+bound3+bound4):(bound1+bound2+bound3+bound4+bound5)]
test4_bert = bert_vec[(bound1+bound2+bound3+bound4+bound5):(bound1+bound2+bound3+bound4+bound5+bound6)]
test5_bert = bert_vec[(bound1+bound2+bound3+bound4+bound5+bound6):]
if not args.infer:
if args.wiki:
dataset_path = Path(utils.config['wikidataset'])
train_dataset = WikipediaDataSet(dataset_path / 'train', word2vec=word2vec, high_granularity=args.high_granularity, sent_bert=train_bert)
dev_dataset = WikipediaDataSet(dataset_path / 'dev', word2vec=word2vec, high_granularity=args.high_granularity, sent_bert=dev_bert)
test_dataset = WikipediaDataSet(dataset_path / 'test', word2vec=word2vec, high_granularity=args.high_granularity, sent_bert=test_bert)
test_dataset_2 = WikipediaDataSet(dataset_path / 'test_cities', word2vec=word2vec, high_granularity=args.high_granularity, sent_bert=test2_bert)
test_dataset_3 = WikipediaDataSet(dataset_path / 'test_elements', word2vec=word2vec, high_granularity=args.high_granularity, sent_bert=test3_bert)
test_dataset_4 = WikipediaDataSet(dataset_path / 'test_clinical', word2vec=word2vec, high_granularity=args.high_granularity, sent_bert=test4_bert)
test_dataset_5 = WikipediaDataSet(dataset_path / 'test_wiki', word2vec=word2vec, high_granularity=args.high_granularity, sent_bert=test5_bert)
else:
dataset_path = Path(utils.config['choidataset'])
train_dataset = ChoiDataset(dataset_path / 'train', word2vec)
dev_dataset = ChoiDataset(dataset_path / 'dev', word2vec)
test_dataset = ChoiDataset(dataset_path / 'test', word2vec)
train_dl = DataLoader(train_dataset, batch_size=args.bs, collate_fn=collate_fn, shuffle=True,
num_workers=args.num_workers)
dev_dl = DataLoader(dev_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
test_dl = DataLoader(test_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
test_dl_2 = DataLoader(test_dataset_2, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
test_dl_3 = DataLoader(test_dataset_3, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
test_dl_4 = DataLoader(test_dataset_4, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
test_dl_5 = DataLoader(test_dataset_5, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
assert bool(args.model) ^ bool(args.load_from) # exactly one of them must be set
if args.model:
model = import_model(args.model)
elif args.load_from:
with open(args.load_from, 'rb') as f:
model = torch.load(f)
model.train()
model = maybe_cuda(model)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
if not args.infer:
best_val_pk = 1.0
for j in range(args.epochs):
train(model, args, j, train_dl, logger, optimizer)
val_pk, threshold = validate(model, args, j, dev_dl, logger)
test_pk = test(model, args, j, test_dl, logger, threshold)
test_pk2 = test(model, args, j, test_dl_2, logger, threshold)
test_pk3 = test(model, args, j, test_dl_3, logger, threshold)
test_pk4 = test(model, args, j, test_dl_4, logger, threshold)
test_pk5 = test(model, args, j, test_dl_5, logger, threshold)
if val_pk < best_val_pk:
logger.debug(
colored(
'Current best model from epoch {} with p_k {} and threshold {}'.format(j, test_pk, threshold),
'green'))
best_val_pk = val_pk
with (checkpoint_path / 'best_model_transformer.t7'.format(j)).open('wb') as f:
torch.save(model, f)
else:
test_dataset = WikipediaDataSet(args.infer, word2vec=word2vec,
high_granularity=args.high_granularity)
test_dl = DataLoader(test_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
print(test(model, args, 0, test_dl, logger, 0.4))
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
import preprocessors as pp
import utils as ut
config = ut.read_config_file('config.yaml')
titanic_pipe = Pipeline(
# complete with the list of steps from the preprocessors file
# and the list of variables from the config
[('categorical_imputer',
pp.CategoricalImputer(
variables=config[2]['Feature_Groups'].get('categorical_vars'))),
('missing_indicator',
pp.MissingIndicator(
variables=config[2]['Feature_Groups'].get('numerical_to_impute'))),
('numerical_imputer',
pp.NumericalImputer(
variables=config[2]['Feature_Groups'].get('numerical_to_impute'))),
('cabin_variable',
pp.ExtractFirstLetter(
variables=config[2]['Feature_Groups'].get('categorical_vars')[1])),
('rare_label_encoder',
pp.RareLabelCategoricalEncoder(
tol=0.05,
variables=config[2]['Feature_Groups'].get('categorical_vars'))),
('categorical_encoder',
pp.CategoricalEncoder(
variables=config[2]['Feature_Groups'].get('categorical_vars'))),
def main():
print "Executing CBO Algo for Equities"
print "-------------------------------"
global kite
global fno_dict, base_dict, config_dict, orders
global scrip_map, sub_list
global order_dict
inst_token = []
#TODO: Add argparser for validating input
if len(sys.argv) < NO_OF_PARAMS:
print "Invalid number of params"
#return
# read config file
config_dict = utils.read_config_file()
# get list of fno
fno_dict = utils.get_fno_dict()
# get yesterdays high low
base_dict = get_yesterdays_ohlc(sys.argv[1])
#get kite object
api_key, access_token, kite = kite_utils.login_kite(None)
# get instrument list, create quote subscription list and
# mapping between instrument token and tradingsymbol
quote_list = []
data = kite.instruments("NSE")
for each in fno_dict:
for instrument in data:
if each == instrument['tradingsymbol']:
entry = "NSE:" + str(instrument['tradingsymbol'])
quote_list.append(entry)
# sub list for subscribing to the quotes
sub_list.append(int(instrument['instrument_token']))
#mapping dictionary for token and trading symbol
scrip_map[int(instrument['instrument_token'])] = str(instrument['tradingsymbol'])
print scrip_map
# open file to write buy/sell orders
fp = open(config_dict['cbo_seed_file'], "w")
# write header
utils.write_header(fp, "CBO")
# Generate order file
count = int(0)
quotes = kite.quote(quote_list)
for each in quotes:
scrip = each.split(":")[1].strip("\n")
if scrip not in base_dict:
continue
if float(quotes[each]["ohlc"]["open"]) < float(config_dict['start_price']):
continue
if float(quotes[each]["ohlc"]["open"]) > float(config_dict['end_price']):
continue
count = int(count) + int(1);
buy, sell = generate_orders(scrip, base_dict[scrip], quotes[each]['ohlc']['open'])
if (buy != None):
fp.write(buy)
if (sell != None):
fp.write(sell)
fp.close()
# create dictionary for active orders
curr_order = kite.orders()
print "------------------------------------------------"
print curr_order
print "------------------------------------------------"
# push all the orders
order_list = []
order_dict = {}
fp = open(config_dict['cbo_seed_file'])
for each in fp:
#ignore line starting with #
if each.startswith("#"):
continue
each = each.rstrip()
line = each.split(" ")
scrip = line[SCRIP_ID]
action = line[ACTION_ID]
price = line[PRICE_ID]
t_price = line[TRIGGER_ID]
target = line[TARGET_ID]
stoploss = line[STOPLOSS_ID]
live_price = line[LIVE_PRICE_ID]
if line[SCRIP_ID] not in order_dict:
order_dict[scrip] = {}
order_dict[scrip][action] = {}
else:
order_dict[scrip][action] = {}
order_dict[scrip][action]['price'] = price
order_dict[scrip][action]['trigger_price'] = t_price
order_dict[scrip][action]['target'] = target
order_dict[scrip][action]['stoploss'] = stoploss
order_dict[scrip][action]['flag'] = 0
order_dict[scrip][action]['live_price'] = live_price
fp.close()
print "----------------------------------------------------------------"
print order_dict
print "----------------- End of order list ----------------------------"
kws = KiteTicker(api_key, access_token, debug=False)
kws.on_ticks = on_ticks
kws.on_connect = on_connect
kws.on_close = on_close
kws.on_error = on_error
kws.on_noreconnect = on_noreconnect
kws.on_reconnect = on_reconnect
kws.on_order_update = on_order_update
kws.connect()
def main(args):
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
file_path = args.input
output_folder_path = args.output
special_delim_sign_path = args.sign
file = open(str(special_delim_sign_path), "r")
special_delim_sign = file.read().encode('utf-8').split("\n")[0]
file.close()
file = open(str(file_path), "r")
raw_content = file.read()
file.close()
result_file_path = None
sentences = [
s for s in raw_content.decode('utf-8').strip().split("\n")
if len(s) > 0 and s != "\n"
]
last_doc_id = 0
last_topic = ""
for sentence in sentences:
first_comma_index = sentence.index(',')
second_comma_index = sentence[first_comma_index + 1:].index(',')
current_doc_id = sentence[0:first_comma_index]
sign_index = sentence.index(special_delim_sign)
start_sentence_index = sign_index + 1
actual_sentence = sentence[start_sentence_index:]
current_topic = sentence[first_comma_index + second_comma_index +
2:sign_index]
if (current_doc_id != last_doc_id):
last_doc_id = current_doc_id
print 'new file index'
print last_doc_id
if (result_file_path != None):
result_file.close()
result_file_path = os.path.join(output_folder_path,
str(current_doc_id) + ".text")
result_file = open(str(result_file_path), "w")
last_topic = ""
if (current_topic != last_topic):
last_topic = current_topic
level = 1 if (current_topic == "TOP-LEVEL SEGMENT") else 2
result_file.write(
(wiki_utils.get_segment_seperator(level, current_topic) +
".").encode('utf-8'))
result_file.write("\n".encode('utf-8'))
if ('\n' in sentence):
print 'back slash in sentnece'
result_file.write(actual_sentence.encode('utf-8'))
#result_file.write(".".encode('utf-8'))
result_file.write("\n".encode('utf-8'))
from data import DataHandler
from models import AENet
import tensorflow as tf
from utils import read_config_file, create_dir
def test_aenet_model(config):
tf.reset_default_graph()
sess = tf.Session()
test_lbs, _ = DataHandler.load_labels(config['test_lbs_file'])
print('Loading test data...done')
config['batch_size'] = test_lbs.shape[0]
aenet = AENet(sess, config, 'AENet', is_train=False)
print('Building AENet model...done')
aenet.restore(config['ckpt_dir'])
print('Loading trained AENet model...done')
print('Testing...')
aenet.deploy(config['result_dir'], test_lbs)
print('Testing...done')
if __name__ == "__main__":
config = read_config_file('./config/JSRT/AENet.cfg')
create_dir(config['result_dir'])
test_aenet_model(config)
import wutils
# By default, all modules will be enabled, examples will be disabled,
# and tests will be disabled.
modules_enabled = ['all_modules']
examples_enabled = False
tests_enabled = False
# Bug 1868: be conservative about -Wstrict-overflow for optimized builds
# on older compilers; it can generate spurious warnings.
cc_version_warn_strict_overflow = ('4', '8', '2')
# Get the information out of the NS-3 configuration file.
config_file_exists = False
(config_file_exists, modules_enabled, examples_enabled, tests_enabled) = read_config_file()
sys.path.insert(0, os.path.abspath('waf-tools'))
try:
import cflags # override the build profiles from waf
finally:
sys.path.pop(0)
cflags.profiles = {
# profile name: [optimization_level, warnings_level, debug_level]
'debug': [0, 2, 3],
'optimized': [3, 2, 1],
'release': [3, 2, 0],
}
cflags.default_profile = 'debug'
def main():
print "Executing CBO Algo for Equities"
print "-------------------------------"
global fno_dict, base_dict, config_dict, orders
inst_token = []
#TODO: Add argparser for validating input
if len(sys.argv) < NO_OF_PARAMS:
print "Invalid number of params"
#return
# read config file
config_dict = utils.read_config_file()
# get list of fno
fno_dict = utils.get_fno_dict()
# get yesterdays high low
base_dict = get_yesterdays_ohlc(sys.argv[1])
'''
#simulate(sys.argv[2])
#return
#open kite connection
if len(sys.argv) == int(NO_OF_PARAMS) + int(1):
request_token = sys.argv[2]
else:
request_token = None
#kite = kite_utils.kite_login(request_token)
config = ConfigParser.ConfigParser()
config.read(config_dict['data_access'])
access_token = config.get('MAIN','DATA_ACCESS_TOKEN')
#my_api = "yvyxm4vynkq1pj8q"
#my_api_secret = "53ekyylrx3orbb85l8isj4o291o22g31"
my_api = str(config_dict['kite_api_key'])
my_api_secret = str(config_dict['kite_api_secret'])
kite = KiteConnect(api_key=my_api)
url = kite.login_url()
# Redirect the user to the login url obtained
# from kite.login_url(), and receive the request_token
# from the registered redirect url after the login flow.
# Once you have the request_token, obtain the access_token
# as follows.
# sys.argv[1] is access token that we get from login
if request_token == None:
kite.set_access_token(access_token)
else:
data = kite.generate_session(request_token, api_secret=my_api_secret)
kite.set_access_token(data["access_token"])
access_token = data["access_token"]
config.set('MAIN','DATA_ACCESS_TOKEN', data["access_token"])
with open(config_dict['data_access'], 'wb') as configfile:
config.write(configfile)
prit kite
'''
kite = kite_utils.login_kite(None)
# get instrument list
quote_list = []
data = kite.instruments("NSE")
for each in fno_dict:
for instrument in data:
if each == instrument['tradingsymbol']:
entry = "NSE:" + str(instrument['tradingsymbol'])
quote_list.append(entry)
# open file to write buy/sell orders
fp = open(config_dict['cbo_seed_file'], "w")
# write header
outstring = "########################################################################################\n"
fp.write(outstring)
outstring = "# CBO file generated for " + time.strftime("%c") +"\n"
fp.write(outstring)
outstring = "########################################################################################\n"
fp.write(outstring)
count = int(0)
quotes = kite.quote(quote_list)
for each in quotes:
scrip = each.split(":")[1].strip("\n")
if float(quotes[each]["ohlc"]["open"]) < float(config_dict['start_price']):
continue
if float(quotes[each]["ohlc"]["open"]) > float(config_dict['end_price']):
continue
count = int(count) + int(1);
buy, sell = generate_orders(scrip, base_dict[scrip], quotes[each]['ohlc']['open'])
if (buy != None):
fp.write(buy)
if (sell != None):
fp.write(sell)
fp.close()
# push all the orders
order_list = []
fp = open(config_dict['cbo_seed_file'])
for each in fp:
#ignore line starting with #
if each.startswith("#"):
continue
each = each.rstrip()
order_list.append(each.split(" "))
fp.close()
print "----------------------------------------------------------------"
print order_list
print "----------------- End of order list ----------------------------"
for each in order_list:
try:
print each
order_id = kite.place_order(
tradingsymbol=str(each[SCRIP_ID]),
exchange="NSE",
transaction_type=str(each[ACTION_ID]),
quantity=1,
order_type="SL",
product="BO",
price = float(each[PRICE_ID]),
trigger_price = float(each[TRIGGER_ID]),
squareoff = float(each[TARGET_ID]),
stoploss = float(each[STOPLOSS_ID]),
variety = "bo",
validity = "DAY"
)
logging.info("Order placed. ID is: {}".format(order_id))
except Exception as e:
logging.info("Order placement failed: {}".format(e.message))
self._error_count = error_count
self._number_failed += no_msgs
if self.config['input']['type'] == 'tailer':
self.input.fail(**args)
else:
error_count = 0
self._number_collected += no_msgs
if self.config['input']['type'] == 'tailer':
self.input.success(**args)
self._error_count = error_count
self._output.close()
def generate_package(self, data):
"""Parse raw data and package the result in required format"""
if self.parser:
data = self.parser.parse(data)
log.debug("Parser %s parsed data %s: ",
self.parser.__class__.__name__, data)
log.debug("Data to be packaged: %s", data)
return generate_payload(data, self.metadata)
if __name__ == "__main__":
# A demo code to show how Collector is used
config = read_config_file()
log = create_logger(__name__, config)
slurm_collector = Collector(config)
slurm_collector.collect()
from flask import Flask, Request
from utils import setup_logger, read_config_file
import os
import rollbar
import rollbar.contrib.flask
from flask import got_request_exception
from celery import Celery
import __builtin__
# Argument parser
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("-c")
args = arg_parser.parse_args()
# Config
cfg = read_config_file(args.c)
# Create App
app = Flask(__name__, static_url_path='')
app.secret_key = SECRET_KEY
app.permanent_session_lifetime = SESSION_DURATION
documentor = Autodoc(app)
app.config['MONGODB_SETTINGS'] = {
'db': cfg["database"]["name"],
'host': cfg["database"]["host"],
'port': cfg["database"]["port"]
}
# Celery
app.config['CELERY_BROKER_URL'] = cfg['redis']['url']
app.config['CELERY_RESULT_BACKEND'] = cfg['redis']['url']
def main(args):
start = timer()
sys.path.append(str(Path(__file__).parent))
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
logger.debug('Running with config %s', utils.config)
print('Running with threshold: ' + str(args.seg_threshold))
preds_stats = utils.predictions_analysis()
if not args.test:
word2vec = gensim.models.KeyedVectors.load_word2vec_format(
utils.config['word2vecfile'], binary=True)
else:
word2vec = None
word2vec_done = timer()
print 'Loading word2vec ellapsed: ' + str(word2vec_done -
start) + ' seconds'
dirname = 'test'
if args.wiki:
dataset_folders = [Path(utils.config['wikidataset']) / dirname]
if (args.wiki_folder):
dataset_folders = []
dataset_folders.append(args.wiki_folder)
print 'running on wikipedia'
else:
if (args.bySegLength):
dataset_folders = getSegmentsFolders(utils.config['choidataset'])
print 'run on choi by segments length'
else:
dataset_folders = [utils.config['choidataset']]
print 'running on Choi'
with open(args.model, 'rb') as f:
model = torch.load(f)
model = maybe_cuda(model)
model.eval()
if (args.naive):
model = naive.create()
for dataset_path in dataset_folders:
if (args.bySegLength):
print 'Segment is ', os.path.basename(dataset_path), " :"
if args.wiki:
if (args.wiki_folder):
dataset = WikipediaDataSet(dataset_path,
word2vec,
folder=True,
high_granularity=False)
else:
dataset = WikipediaDataSet(dataset_path,
word2vec,
high_granularity=False)
else:
dataset = ChoiDataset(dataset_path, word2vec)
dl = DataLoader(dataset,
batch_size=args.bs,
collate_fn=collate_fn,
shuffle=False)
with tqdm(desc='Testing', total=len(dl)) as pbar:
total_accurate = 0
total_count = 0
total_loss = 0
acc = accuracy.Accuracy()
for i, (data, targets, paths) in enumerate(dl):
if i == args.stop_after:
break
pbar.update()
output = model(data)
targets_var = Variable(maybe_cuda(torch.cat(targets, 0),
args.cuda),
requires_grad=False)
batch_loss = 0
output_prob = softmax(output.data.cpu().numpy())
output_seg = output_prob[:, 1] > args.seg_threshold
target_seg = targets_var.data.cpu().numpy()
batch_accurate = (output_seg == target_seg).sum()
total_accurate += batch_accurate
total_count += len(target_seg)
total_loss += batch_loss
preds_stats.add(output_seg, target_seg)
current_target_idx = 0
for k, t in enumerate(targets):
document_sentence_count = len(t)
sentences_length = [s.size()[0] for s in data[k]
] if args.calc_word else None
to_idx = int(current_target_idx + document_sentence_count)
h = output_seg[current_target_idx:to_idx]
# hypothesis and targets are missing classification of last sentence, and therefore we will add
# 1 for both
h = np.append(h, [1])
t = np.append(t.cpu().numpy(), [1])
acc.update(h, t, sentences_length=sentences_length)
current_target_idx = to_idx
logger.debug('Batch %s - error %7.4f, Accuracy: %7.4f', i,
batch_loss, batch_accurate / len(target_seg))
pbar.set_description('Testing, Accuracy={:.4}'.format(
batch_accurate / len(target_seg)))
average_loss = total_loss / len(dl)
average_accuracy = total_accurate / total_count
calculated_pk, _ = acc.calc_accuracy()
logger.info('Finished testing.')
logger.info('Average loss: %s', average_loss)
logger.info('Average accuracy: %s', average_accuracy)
logger.info('Pk: {:.4}.'.format(calculated_pk))
logger.info('F1: {:.4}.'.format(preds_stats.get_f1()))
end = timer()
print('Seconds to execute to whole flow: ' + str(end - start))
from argparse import ArgumentParser
from utils import config, read_config_file
parser = ArgumentParser()
parser.add_argument('--cuda', help='Is cuda?', action='store_true')
parser.add_argument('--model', help='Model file path', required=True)
parser.add_argument('--config',
help='Path to config.json',
default='config.json')
parser.add_argument('--test', help='Use fake word2vec', action='store_true')
parser.add_argument('--port', type=int, help='List to this port')
args = parser.parse_args()
read_config_file(args.config)
config.update(args.__dict__)
from webapp import app
app.run(debug=True, port=args.port)
from flask import Flask, Request
from utils import setup_logger, read_config_file
import os
import rollbar
import rollbar.contrib.flask
from flask import got_request_exception
from celery import Celery
import __builtin__
# Argument parser
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("-c")
args = arg_parser.parse_args()
# Config
cfg = read_config_file(args.c)
# Create App
app = Flask(__name__, static_url_path='')
app.secret_key = SECRET_KEY
app.permanent_session_lifetime = SESSION_DURATION
documentor = Autodoc(app)
app.config['MONGODB_SETTINGS'] = {
'db': cfg["database"]["name"],
'host': cfg["database"]["host"],
'port': cfg["database"]["port"]
}
# Celery
app.config['CELERY_BROKER_URL'] = cfg['redis']['url']
app.config['CELERY_RESULT_BACKEND'] = cfg['redis']['url']
def main(args):
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
file_path = args.input
segments_path = args.segment
output_folder_path = args.output
file = open(str(segments_path), "r")
segments_content = file.read()
file.close()
file = open(str(file_path), "r")
raw_content = file.read()
file.close()
sentences = [
s for s in raw_content.decode('utf-8').strip().split("\n")
if len(s) > 0 and s != "\n"
]
segments = [
s for s in segments_content.decode('utf-8').strip().split("\n")
if len(s) > 0 and s != "\n"
]
result_file_path = None
last_doc_id = 0
last_topic = ""
if (len(sentences) != len(segments)):
print "len(sentences) != len(segments)"
return
for i in range(len(sentences)):
sentence = sentences[i]
segment = segments[i].encode('utf-8').split("\r")[0]
first_comma_index = segment.index(',')
second_comma_index = segment[first_comma_index + 1:].index(',')
current_doc_id = segment[0:first_comma_index]
current_topic = segment[first_comma_index + second_comma_index + 2:]
if (current_doc_id != last_doc_id):
last_doc_id = current_doc_id
print 'new file index'
print last_doc_id
if (result_file_path != None):
result_file.close()
result_file_path = os.path.join(output_folder_path,
str(current_doc_id) + ".text")
result_file = open(str(result_file_path), "w")
last_topic = ""
if (current_topic != last_topic):
last_topic = current_topic
level = 1 if (current_topic == "TOP-LEVEL SEGMENT") else 2
result_file.write(
(wiki_utils.get_segment_seperator(level, current_topic) +
".").encode('utf-8'))
result_file.write("\n".encode('utf-8'))
actual_sentence = sentence
result_file.write(actual_sentence.encode('utf-8'))
if ('\n' in sentence):
print 'back slash in sentnece'
#result_file.write(".".encode('utf-8'))
result_file.write("\n".encode('utf-8'))
