def parse_args_and_setup():
# parse and print arguments
utils.parse_arguments()
if utils.global_args.mode == 'stats':
utils.quit('stats mode not supported in data_analytics program; only output mode supported')
if utils.global_args.mode == 'input':
utils.quit('input mode not supported in data_analytics program; only output mode supported')
return
def data_input_parse_args():
# parse and print arguments
utils.parse_arguments()
if utils.global_args.mode == 'stats':
utils.quit('stats mode not supported in data_input program; only input mode supported')
if utils.global_args.mode == 'output':
utils.quit('output mode not supported in data_input program; only input mode supported')
if utils.global_args.inputxls == "" and utils.global_args.inputxml == "":
utils.quit('both input excel file and xml file cannot be empty')
def create_cpp_ros_package():
(package, depends) = utils.parse_arguments(['roscpp','std_msgs'])
# Make directories
p = os.path.abspath(package)
os.makedirs(os.path.join(p,"src"))
print "Created package directory 'src'"
os.makedirs(os.path.join(p,"src","lib"))
print "Created package directory 'src/lib'"
os.makedirs(os.path.join(p,"include"))
print "Created package directory 'include'"
os.makedirs(os.path.join(p,"include",package))
print "Created package directory 'include/%s'"%package
os.makedirs(os.path.join(p,"launch"))
print "Created package directory 'launch'"
# Text files
manifest_depends = ''.join([' <depend package="%s"/>\n'%d for d in depends])
cmake_depends = ''.join(['%s '%d for d in depends])
templates = get_templates(package)
for filename, template in templates.iteritems():
contents = utils.instantiate_template(template, package, package, package, utils.author_name(), manifest_depends, cmake_depends)
try:
p = os.path.abspath(os.path.join(package, filename))
f = open(p, 'w')
f.write(contents.encode('utf-8'))
print "Created package file", p
finally:
f.close()
utils.print_concluding_catkin_message(package)
def main():
global output, events, file_name
args = parse_arguments()
output = os.environ['OUTPUT'] if 'OUTPUT' in os.environ else args.output
start = os.environ['START'] if 'START' in os.environ else args.start
start_date = datetime.strptime(start, '%Y-%m-%d')
end = os.environ['END'] if 'END' in os.environ else args.end
final_end_date = end_date = datetime.strptime(end, '%Y-%m-%d')
days = (end_date - start_date).days
file_name = 'osu-events-from-{}-to-{}'.format(start, end)
events = []
while days > 365:
end_date = start_date + timedelta(days=365)
end = end_date.strftime('%Y-%m-%d')
events += get_events(start, end)
start_date = datetime.strptime(end, '%Y-%m-%d') + timedelta(days=1)
start = start_date.strftime('%Y-%m-%d')
end_date = final_end_date
end = end_date.strftime('%Y-%m-%d')
days = (end_date - start_date).days
events += get_events(start, end)
fields = [event_filter for event_filter in send_request(EVENT_FILTERS_URL).keys()]
fields.append('departments')
create_report_by(fields)
def create_ros_package(type):
is_catkin=True
if type == 'ros-legacy':
is_catkin=False
(package, depends) = utils.parse_arguments([], is_catkin)
p = os.path.abspath(package)
os.makedirs(p)
manifest_depends = ''.join([' <depend package="%s"/>\n'%d for d in depends])
cmake_depends = ''.join(['%s '%d for d in depends])
p = os.path.abspath(package)
templates = get_ros_text_templates(type)
for filename, template in templates.iteritems():
contents = utils.instantiate_template(template, package, package, package, utils.author_name(), manifest_depends, cmake_depends)
try:
p = os.path.abspath(os.path.join(package, filename))
f = open(p, 'w')
f.write(contents.encode('utf-8'))
print "Created package file", p
finally:
f.close()
if type == 'ros-legacy':
utils.print_concluding_message(package)
else:
utils.print_concluding_catkin_message(package)
def create_winros_catkin_package():
(package, depends) = utils.parse_arguments([])
if 'roscpp' in depends:
create_cpp_package_directory(package)
templates = get_cpp_templates(package)
elif 'rospy' in depends:
create_py_package_directory(package)
templates = get_py_templates(package)
else:
create_common_package_directory(package)
templates = get_common_templates(package)
build_depends = ''.join(
[' <build_depend>%s</build_depend>\n' % d for d in depends])
run_depends = ''.join(
[' <run_depend>%s</run_depend>\n' % d for d in depends])
cmake_depends = ''.join(['%s ' % d for d in depends])
for filename, template in templates.iteritems():
contents = utils.instantiate_template(template, package, package,
utils.author_name(),
build_depends, run_depends,
cmake_depends)
try:
p = os.path.abspath(os.path.join(package, filename))
f = open(p, 'w')
f.write(contents.encode('utf-8'))
print "Created package file", p
finally:
f.close()
print "\nPlease edit package.xml, mainpage.dox and CMakeLists.txt to finish creating your package."
def main(arguments):
"""Main run code for processing embeddings."""
# Parse the arguments
args = utils.parse_arguments(arguments)
# load the embeddings data object
path_to_embeddings = args.infile + '.embedded.pkl'
embeddings, _ = utils.load_data(path_to_embeddings, 'dict')
tf.logging.log(
tf.logging.INFO,
'{} lines in embeddings: {}'.format(len(embeddings.keys()),
path_to_embeddings))
all_embeddings = []
with open(path_to_embeddings + '_unique_strings.csv', 'wb') as outfile:
for k, v in embeddings.items():
output_line = v['line'].encode('utf-8')
if args.pairs:
output_line_response = v['response'].encode('utf-8')
outfile.write(output_line + args.delimiter.encode('utf-8') +
output_line_response + b'\n')
else:
outfile.write(output_line + b'\n')
all_embeddings.append(v['line_embedding'])
# Convert to a numpy array
all_embeddings_np = np.array([np.array(xi) for xi in all_embeddings])
array_outfile = path_to_embeddings + '_unique_strings_embeddings.txt'
np.savetxt(array_outfile, all_embeddings_np)
# Log the embedding shape
tf.logging.log(tf.logging.INFO,
'Embedings shape {}'.format(all_embeddings_np.shape))
def main(arguments):
"""Main run function to process the pairs data."""
# Parse the arguments
args = utils.parse_arguments(arguments)
tf.logging.info('Select and save {} random pairs...'.format(
args.num_lines))
tf.logging.info('Input file: {}'.format(args.infile))
# load the lines
lines, _ = utils.load_data(args.infile, dest_type='list', delimiter='\n')
tf.logging.info("Loaded {} lines: {}".format(len(lines), args.infile))
with open(args.outfile, 'w', encoding='iso-8859-1') as outputfile:
writer = csv.writer(outputfile, delimiter=args.delimiter)
collected_pairs = utils.extract_pairs_from_lines(lines)
random_idxs = np.random.choice(len(collected_pairs),
args.num_lines,
replace=False)
for random_id in random_idxs:
pair = collected_pairs[random_id]
writer.writerow(pair)
tf.logging.info('Wrote {} pairs to {}.'.format(args.num_lines,
args.outfile))
def __init__(self):
self.args = utils.parse_arguments()
chrome_application_path = utils.get_application_path()
chrome_application_directory = os.path.dirname(chrome_application_path)
self.ipc_fuzzer_binary = utils.get_fuzzer_application_name()
self.ipc_fuzzer_binary_path = os.path.join(
chrome_application_directory, self.ipc_fuzzer_binary)
def __init__(self):
self.args = utils.parse_arguments()
chrome_application_path = utils.get_application_path()
chrome_application_directory = os.path.dirname(chrome_application_path)
self.ipc_fuzzer_binary = utils.get_fuzzer_application_name()
self.ipc_fuzzer_binary_path = os.path.join(
chrome_application_directory, self.ipc_fuzzer_binary)
def main():
args = parse_arguments()
image_model_name = args.image_model_name
embedding_type = args.embedding_type
embedding_dim = args.embedding_dim
batch_size = args.batch_size
train(image_model_name, embedding_type, embedding_dim, batch_size)
def update_event(em, user_id, msg, all_ids, all_names, user_email='', user_name='', create=False):
'''
create an event from the message.
and add to event manager
'''
arg_dict = parse_arguments(msg)
if create:
event = Event(user_id, user_email, user_name)
else:
event = em.find_event(user_id)
if '-guests' in arg_dict:
guests = parse_mentions(arg_dict['-guests'])
# check if valid guests
for guest in guests:
if guest not in all_ids:
print("Error invalid guests")
return em, False
# find relevant users' id, email, name
ids = []
names = []
for id, name in zip(all_ids, all_names):
if id in guests:
ids.append(id)
names.append(name)
# get guest ids and names
event.set_guests(ids)
#event.set_guest_emails(emails)
event.set_guest_names(names)
if '-title' in arg_dict:
event.set_title(arg_dict['-title'])
if '-date' in arg_dict:
event.set_date(arg_dict['-date'])
if '-time' in arg_dict:
event.set_time(arg_dict['-time'])
if '-len' in arg_dict:
event.set_length(arg_dict['-len'])
if '-loc' in arg_dict:
event.set_loc(arg_dict['-loc'])
if '-alert' in arg_dict:
event.set_alert(arg_dict['-alert'])
if '-desc' in arg_dict:
event.set_desc(arg_dict['-desc'])
em.add_event(user_id, event)
return em, True
def __init__(self):
self.args = utils.parse_arguments()
self.ipc_message_util_binary = utils.application_name_for_platform(
IPC_MESSAGE_UTIL_APPLICATION)
self.ipc_fuzzer_binary = utils.get_fuzzer_application_name()
self.ipc_message_util_binary_path = utils.get_application_path(
self.ipc_message_util_binary)
self.ipc_fuzzer_binary_path = utils.get_application_path(
self.ipc_fuzzer_binary)
def __init__(self):
self.args = utils.parse_arguments()
self.ipc_message_util_binary = utils.application_name_for_platform(
IPC_MESSAGE_UTIL_APPLICATION)
self.ipc_fuzzer_binary = utils.get_fuzzer_application_name()
self.ipc_message_util_binary_path = utils.get_application_path(
self.ipc_message_util_binary)
self.ipc_fuzzer_binary_path = utils.get_application_path(
self.ipc_fuzzer_binary)
def main():
args = parse_arguments()
weights_file = args.weights_file[0]
image_model_name = args.image_model_name
embedding_type = args.embedding_type
embedding_dim = args.embedding_dim
batch_size = args.batch_size
evaluate(weights_file, image_model_name, embedding_type, embedding_dim,
batch_size)
def main():
args = parse_arguments()
setup_spark()
df_u, df_i, df_r = read_file(args)
print('='*100 + args.users_path)
df_u.show()
df_u.printSchema()
df_u.select('Age').summary().show()
df_u.createOrReplaceTempView("users")
df2 = sc.sql("select count(*) from (select Age, count(*) from users group by Age) as a")
df2.show()
print('='*100 + args.users_path)
df_i.show()
def main(arguments):
"""Main run function for interacting with the model."""
# Parse the arguments
args = utils.parse_arguments(arguments)
tf.logging.info('Loading unique strings.')
data_path = args.infile
unique_strings_path = data_path + '.embedded.pkl_unique_strings.csv'
# load the unique lines
with open(unique_strings_path) as f:
unique_strings = [line.rstrip() for line in f]
tf.logging.info('Lodaded {} unique strings'.format(len(unique_strings)))
# define the path of the nearest neighbor model to use
annoy_index_path = data_path + '.ann'
# Load generative models from pickles to generate from scratch.
try:
tf.logging.info('Build generative model...')
gen_model_use = utils.GenModelUSE(
annoy_index_path=annoy_index_path,
unique_strings=unique_strings,
module_path=args.module_path,
use_sentence_piece=args.use_sentence_piece)
tf.logging.info('Generative model built.')
except (OSError, IOError) as e:
tf.logging.error(e)
tf.logging.info('Error building generative model.')
# build a loop for interactive mode
while True:
# get user input
user_input = input('\nQuery Text: ')
# if user input is too short
if len(user_input) < 1:
continue
nns, distances = gen_model_use.inference(
user_input,
num_neighbors=args.num_neighbors,
use_sentence_piece=args.use_sentence_piece)
# print all the returned responses, and distance to input
for nn, distance in zip(nns, distances):
print('d: {}, {}'.format(distance,
unique_strings[nn].split(args.delimiter)))
def main():
args = parse_arguments()
algo = args.algo
env = gym.make("TwoLinkArm-v0")
if (algo == "lqr"):
plot_path = os.path.join(os.getcwd(), "lqr_plots")
make_dir(plot_path)
x0, u_seq = run_lqr(env, plot_path)
elif (algo == "ilqr"):
plot_path = os.path.join(os.getcwd(), "ilqr_plots")
make_dir(plot_path)
run_ilqr(env, plot_path)
else:
raise ValueError("algo could only be \"lqr\" or \"ilqr\" ")
def main(arguments):
"""Main run function for interacting with the model."""
# Parse the arguments
args = utils.parse_arguments(arguments)
tf.logging.info('Loading unique strings.')
data_path = args.infile
unique_strings_path = data_path + '.embedded.pkl_unique_strings.csv'
# load the unique lines
with open(unique_strings_path) as f:
unique_strings = [line.rstrip() for line in f]
tf.logging.info('Lodaded {} unique strings'.format(len(unique_strings)))
# define the path of the nearest neighbor model to use
annoy_index_path = data_path + '.ann'
# Load generative models from pickles to generate from scratch.
try:
tf.logging.info('Build generative model...')
gen_model_use = utils.GenModelUSE(
annoy_index_path=annoy_index_path,
unique_strings=unique_strings,
module_path=args.module_path,
use_sentence_piece=args.use_sentence_piece)
tf.logging.info('Generative model built.')
except (OSError, IOError) as e:
tf.logging.info('Error building generative model.')
# build a loop for interactive mode
while True:
# get user input
user_input = input('\nQuery Text: ')
# if user input is too short
if len(user_input) < 1:
continue
resp = gen_model_use.inference(user_input, num_neighbors=1,
args=args).split(args.delimiter)
# respond with the response from the [input,response] pair
# using matching to the input string
tf.logging.info('Closest matched root: {}'.format(resp[0]))
if len(resp) > 0:
tf.logging.info('Response: {}'.format(resp[0]))
def main():
try:
args = parse_arguments()
main_logger.info(args)
if args.mode == SCAN:
run_scan(args)
elif args.mode == REPORTS:
get_reports(args)
elif args.mode == DEPCHECK:
depcheck(args)
except Exception as e:
main_logger.info(e)
try:
run_subprocess(f"docker network rm {NETWORK_SCAN}")
except Exception as _:
main_logger.warning(f"Error removing {NETWORK_SCAN}")
try:
run_subprocess(f"docker volume rm {VOL_SCAN}")
except Exception as _:
main_logger.warning(f"Error removing {VOL_SCAN}")
def create_winros_catkin_package():
(package, depends) = utils.parse_arguments(['std_msgs'])
create_package_directory(package)
templates = get_templates(package)
build_depends = ''.join(
[' <build_depend>%s</build_depend>\n' % d for d in depends])
run_depends = ''.join(
[' <run_depend>%s</run_depend>\n' % d for d in depends])
cmake_depends = ''.join(['%s ' % d for d in depends])
for filename, template in templates.iteritems():
contents = utils.instantiate_template(template, package, package,
utils.author_name(),
build_depends, run_depends,
cmake_depends)
try:
p = os.path.abspath(os.path.join(package, filename))
f = open(p, 'w')
f.write(contents.encode('utf-8'))
print "Created package file", p
finally:
f.close()
print "\nPlease edit package.xml, mainpage.dox, CMakeLists.txt, and add the package subdirectory."
def create_qt_ros_package(type):
is_catkin=True
if type == 'qt-ros-legacy':
is_catkin=False
#print "Create qt_ros_package(is_catkin:%r)" % is_catkin
(package, depends) = utils.parse_arguments(['qt_build','roscpp'], is_catkin)
# Make directories
p = os.path.abspath(package)
os.makedirs(os.path.join(p,"src"))
os.makedirs(os.path.join(p,"include"))
os.makedirs(os.path.join(p,"include",package))
os.makedirs(os.path.join(p,"resources"))
os.makedirs(os.path.join(p,"resources","images"))
os.makedirs(os.path.join(p,"ui"))
print "Created qt package directories."
# Qt text files
manifest_depends = ''.join([' <depend package="%s"/>\n'%d for d in depends])
cmake_depends = ''.join(['%s '%d for d in depends])
templates = get_qt_text_templates(package, type)
for filename, template in templates.iteritems():
contents = utils.instantiate_template(template, package, package, package, utils.author_name(), manifest_depends, cmake_depends)
try:
p = os.path.abspath(os.path.join(package, filename))
f = open(p, 'w')
f.write(contents.encode('utf-8'))
print "Created package file", p
finally:
f.close()
# Qt binary files
template_dir = os.path.join(os.path.dirname(__file__),'templates',type)
shutil.copy(os.path.join(template_dir,'resources','images','icon.png'),
os.path.join(os.path.abspath(package),'resources','images','icon.png'))
if type == 'qt-ros-legacy':
utils.print_concluding_message(package)
else:
utils.print_concluding_catkin_message(package)
def main():
args = utils.parse_arguments()
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
print(device)
with np.printoptions(precision=4, suppress=True):
if args.experiment == 'mlp':
train_acc, val_acc, test_acc = run_mlp_experiment(args, device)
elif args.experiment == 'vgg':
train_acc, val_acc, test_acc = run_vgg_experiment(args, device)
else:
raise NotImplementedError(
'experiment must be mlp or vgg, but %s was given' %
args.experiment)
print('Final validation accuracy: %f' % val_acc[-1])
print('Job finished')
if args.save_results:
np.save(args.results_filename + '_train_acc', train_acc)
np.save(args.results_filename + '_val_acc', val_acc)
np.save(args.results_filename + '_test_acc', test_acc)
def main(arguments):
"""Main run function for indexing the embeddings."""
# Parse the arguments
args = utils.parse_arguments(arguments)
unique_strings_path = args.infile + '.embedded.pkl_unique_strings.csv'
# Load the unique lines
with open(unique_strings_path) as f:
unique_strings = [line.rstrip() for line in f]
unique_embeddings_path = (args.infile +
'.embedded.pkl_unique_strings_embeddings.txt')
# Load the unique embeddings
with open(unique_embeddings_path) as f:
unique_embeddings = [[float(x) for x in line.strip().split()]
for line in f]
tf.logging.info(
'Loaded {} unique strings, {} embeddings of dimension {}'.format(
len(unique_strings), len(unique_embeddings),
len(unique_embeddings[0])))
# Length of item vector that will be indexed
nn_forest = AnnoyIndex(512)
for i in range(len(unique_strings)):
v = unique_embeddings[i]
nn_forest.add_item(i, v)
# Build an approximate nearest neighbor forest with num_trees
nn_forest.build(int(args.num_trees))
output_path = args.infile + '.ann'
nn_forest.save(output_path)
tf.logging.info('Index forest built {}'.format(output_path))
def create_comms_package(type):
(package, depends) = utils.parse_arguments(['std_msgs'])
# Make directories
p = os.path.abspath(package)
os.makedirs(os.path.join(p,"msg"))
os.makedirs(os.path.join(p,"srv"))
print "Created package directories."
# Text files
manifest_depends = ''.join([' <depend package="%s"/>\n'%d for d in depends])
cmake_depends = ''.join(['%s '%d for d in depends])
templates = get_text_templates(package, type)
for filename, template in templates.iteritems():
contents = utils.instantiate_template(template, package, package, package, utils.author_name(), manifest_depends, cmake_depends)
try:
p = os.path.abspath(os.path.join(package, filename))
f = open(p, 'w')
f.write(contents.encode('utf-8'))
print "Created package file", p
finally:
f.close()
utils.print_concluding_catkin_message(package)
def create_validation():
opt = parse_arguments()
print(opt)
root = opt.root
save_dir = root + 'val_dataset_peragent/'
if not os.path.exists(save_dir):
os.mkdir(save_dir)
else:
print("Data folder exists! Continue?")
pdb.set_trace()
loader = features_loader(root=root,
probs=opt.probs,
norm=opt.norm,
ours=opt.ours,
partition='train/')
val_z, val_images_indexes_sender, val_images_indexes_receiver = \
create_val_batch(opt, loader)
suffix = 'seed%d_same%d' % (opt.manualSeed, opt.same)
pickle.dump(val_z, open(save_dir + "val_z" + suffix, "wb"))
pickle.dump(val_images_indexes_sender,
open(save_dir + "val_images_indexes_sender" + suffix, "wb"))
pickle.dump(val_images_indexes_receiver,
open(save_dir + "val_images_indexes_receiver" + suffix, "wb"))
best_loss = min(val_loss, best_loss)
# Save checkpoint
model_filename = 'checkpoint_%03d.pth.tar' % epoch
checkpoint = {
'epoch': epoch,
'model': generator.state_dict(),
'discriminator': discriminator.state_dict(),
'gen_optimizer': gen_optimizer.state_dict(),
'disc_optimizer': disc_optimizer.state_dict(),
'best_loss': best_loss
}
save_checkpoint(checkpoint, model_filename, is_best, save_path)
# Evaluate model using PSNR and SSIM metrics
evaluate(args, generator, val_loader)
def save_checkpoint(checkpoint, filename, is_best, save_path):
print("===> Saving checkpoint '{}'".format(filename))
model_filename = save_path / filename
best_filename = save_path / 'model_best.pth.tar'
torch.save(checkpoint, model_filename)
if is_best:
shutil.copyfile(model_filename, best_filename)
print("===> Saved checkpoint '{}'".format(model_filename))
if __name__ == '__main__':
main(parse_arguments(argv[1] if len(argv) >= 2 else "run_configs/default_gan.ini"))
# Test case: GET /spendingauthority with invalid authority onids
def test_get_non_authority_onids(self, endpoint='/spendingauthority'):
for onid in self.test_cases['invalid_authority_onids']:
params = {'onid': onid}
response = utils.make_request(self, endpoint, 200, params=params)
spending_schema = []
self.assertEqual(response.json()['data'], spending_schema)
# Test case: GET /spendingauthority with bad request
def test_bad_request_response(self, endpoint='/spendingauthority'):
bad_params = [{'onid': ''}, {}]
for param in bad_params:
response = utils.make_request(self, endpoint, 400, params=param)
error_schema = utils.get_resource_schema(self, 'Error')
utils.check_schema(self, response, error_schema)
if __name__ == '__main__':
arguments, argv = utils.parse_arguments()
# Setup logging level
if arguments.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
integration_tests.setup(arguments.config_path, arguments.openapi_path)
unittest.main(argv=argv)
integration_tests.cleanup()
def output(last_matches):
music_cluster_list = []
for idx, cluster in enumerate(last_matches):
for music in cluster:
music_cluster_list.append((music, idx))
music_cluster_list.sort()
for t in music_cluster_list:
print t[0], t[1]
if __name__ == '__main__':
usage_text = 'Usage: %prog -i input_file -k num_of_clusters [-c centroids_file]'
parser = OptionParser(usage=usage_text)
(options, args) = parse_arguments(parser)
file_name = options.input
k = options.k
tags = get_set_of_tags_from_file(file_name)
list_of_all_tags = sorted(tags)
musics = get_musics_from_file(file_name, list_of_all_tags)
if options.debug:
print 'Numbers of tags:', len(list_of_all_tags)
if options.centroids_file:
centroids = get_centroids_from_file(musics, k, options.centroids_file)
else:
# centroids = random_centroids(k)
centroids = forgy_initialization(musics, k)
def main():
"""Runs queries to create training and prediction tables from clean data."""
# Load config shared by all steps of feature creation.
config_path = utils.parse_arguments(sys.argv).config_path
config = utils.read_config(config_path)
# Project-wide config.
global_config = config['global']
# Path to SQL files.
queries_path = config['file_paths']['queries']
# SQL files for different pipeline steps.
query_files = config['query_files']
# Parameters unique to individual pipeline steps.
query_params = config['query_params']
# Create the dataset to hold data for the pipeline run.
utils.create_dataset(
destination_project=global_config['destination_project_id'],
destination_dataset=global_config['destination_dataset'])
# Query to remove nulls from the target column (company_response_to_consumer)
# and from complaint_narrative column.
remove_nulls_params = utils.merge_dicts(global_config,
query_params['remove_nulls'])
utils.create_table(
query_path=os.path.join(queries_path, query_files['remove_nulls']),
query_params=remove_nulls_params,
destination_project=global_config['destination_project_id'],
destination_dataset=global_config['destination_dataset'],
destination_table=global_config['nulls_removed_table'],
partition_field=None)
# Query to cleanup the categories of issue, subissue, product, subproduct.
utils.create_table(
query_path=os.path.join(queries_path, query_files['clean_categories']),
query_params=global_config,
destination_project=global_config['destination_project_id'],
destination_dataset=global_config['destination_dataset'],
destination_table=global_config['cleaned_features_table'],
partition_field=None)
# Query to merge the cleaned features and the table with nulls removed.
utils.create_table(
query_path=os.path.join(queries_path, query_files['combine_tables']),
query_params=global_config,
destination_project=global_config['destination_project_id'],
destination_dataset=global_config['destination_dataset'],
destination_table=global_config['clean_table'],
partition_field=None)
# Query to split the clean dataset into training and prediction datasets.
# The training dataset will be fed to the AutoML Tables for training and
# the prediction dataset will be used for batch prediction.
features_split_params = utils.merge_dicts(
global_config, query_params['train_predict_split'])
utils.create_table(
query_path=os.path.join(queries_path,
query_files['train_predict_split']),
query_params=features_split_params,
destination_project=global_config['destination_project_id'],
destination_dataset=global_config['destination_dataset'],
destination_table=global_config['train_predict_split'],
partition_field=None)
# Query to create the prediction table.
features_split_params = utils.merge_dicts(
global_config, query_params['train_predict_split'])
utils.create_table(
query_path=os.path.join(queries_path,
query_files['prediction_features']),
query_params=features_split_params,
destination_project=global_config['destination_project_id'],
destination_dataset=global_config['destination_dataset'],
destination_table=global_config['features_predict_table'],
partition_field=None)
# Query to create the training table along with the manual split into train,
# validation and test rows for the AutoML tables.
features_split_params = utils.merge_dicts(
global_config, query_params['train_predict_split'])
utils.create_table(
query_path=os.path.join(queries_path,
query_files['training_features']),
query_params=features_split_params,
destination_project=global_config['destination_project_id'],
destination_dataset=global_config['destination_dataset'],
destination_table=global_config['features_train_table'],
partition_field=None)
""" Gather all parameters (source image, layer descriptor and octave),
create a net and start to dream. """
source_path = get_source_image(args)
layer = get_layer_descriptor(args)
octave = (args.octaves if args.octaves else randint(1, 9))
model_file = '../caffe/models/bvlc_googlenet/bvlc_googlenet.caffemodel'
if args.network:
create_net(model_file)
net = load_net(model_file)
psycam = PsyCam(net=net)
psycam.iterated_dream(source_path=source_path,
end=layer, octaves=octave)
if __name__ == "__main__":
try:
args = parse_arguments(sys.argv[1:])
while True:
start_dream(args)
if not args.continually:
break
except Exception as e:
import traceback
print(traceback.format_exc())
print('Quitting PsyCam')
def __init__(self):
self.args = utils.parse_arguments()
self.ipc_fuzzer_binary = utils.get_fuzzer_application_name()
self.ipc_fuzzer_binary_path = utils.get_application_path(
self.ipc_fuzzer_binary)
if __name__ == '__main__':
import datetime as dt
import multiprocessing as mp
from utils import parse_arguments
from InterventionsMIP import load_config_file, logger, change_paths
# Parse arguments
args = parse_arguments()
# Load config file
load_config_file(args.f_config)
# Adjust paths
change_paths(args)
from instances import load_instance, load_tiers, load_seeds
from policy_search_functions import policy_search, capacity_policy_search
from objective_functions import multi_tier_objective, multi_tier_objective_ACS
from policies import MultiTierPolicy as MTP
from policies import MultiTierPolicy_ACS as MTP_ACS
# Parse city and get corresponding instance
instance = load_instance(args.city,
setup_file_name=args.f,
transmission_file_name=args.tr,
hospitalization_file_name=args.hos)
train_seeds, test_seeds = load_seeds(args.city, args.seed)
tiers = load_tiers(args.city, tier_file_name=args.t)
# TODO Read command line args for n_proc for better integration with crunch
n_proc = args.n_proc
# TODO: pull out n_replicas_train and n_replicas_test to a config file
import numpy as np
from datetime import datetime
from decouple import config
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.feature_extraction.text import TfidfVectorizer
from hdfs.ext.kerberos import KerberosClient
from utils import (
clean_text,
get_keys,
parse_arguments,
)
from models import (OneVsRestLogisticRegression)
from queries import (get_train_data, get_list_of_classes)
options = parse_arguments()
URL_ORACLE_SERVER = options['oracle_server']
USER_ORACLE = options['oracle_user']
PASSWD_ORACLE = options['oracle_password']
ORACLE_DRIVER_PATH = options['oracle_driver_path']
HDFS_URL = options['hdfs_url']
HDFS_USER = options['hdfs_user']
HDFS_MODEL_DIR = options['hdfs_model_dir']
START_DATE = options['start_date']
END_DATE = options['end_date']
NEGATIVE_CLASS_VALUE = 13
ID_COLUMN = 'SNCA_DK'
TEXT_COLUMN = 'SNCA_DS_FATO'
LABEL_COLUMN = 'DMDE_MDEC_DK'
def main():
logfile = "{0}/log/sentence_segment.log".format(PROJECT_DIR)
utils.setup_logging(logfile, logging.INFO)
logging.info(sys.path)
args = utils.parse_arguments()
batchsize = args.batchsize
start_time = time.time()
# set_env_vars()
spark_session = spark.create_spark_session()
es = elastic.check_elasticsearch()
es_write_conf = spark.broadcast_es_write_config(spark_session)
s3resource = aws.create_s3_resource()
keys = aws.get_list_s3_files(s3resource,
filetype=TEXT_FOLDER,
numrows=batchsize)
pbooks = s3_to_rdd(spark_session, keys)
# testing_rdd = spark.sparkContext.wholeTextFiles("s3a://jason-b/{0}".format(TEXT_FOLDER), minPartitions=6, use_unicode=False)
# spark.log_rdd(pbooks)
pipeline = spark_nlp.setup_pipeline()
books = spark_nlp.segment_sentences(spark_session, pbooks, pipeline)
# Go from one book per row to one sentence per row
sentences = books.select(
func.monotonically_increasing_id().alias("sentence_id"),
func.col("fileName"),
func.posexplode("sentence.result").alias("position", "sentenceText"),
func.size("sentence.result").alias("numSentencesInBook"),
)
# logging.info("Num Sentences: {0}".format(sentences.count()))
sentences = spark_nlp.tokenize_sentences(sentences)
count_syllables_udf = func.udf(
lambda s: _udf_sentence_count_syllables_sentence(s),
ArrayType(IntegerType()))
count_sentence_multisyllables_udf = func.udf(
lambda s: sum(_udf_sentence_count_syllables_sentence(s)),
IntegerType())
count_array_multisyllables_udf = func.udf(
lambda a: sum(_udf_array_count_syllables_sentence(a)), IntegerType())
sentences = sentences.select(
"sentence_id",
"fileName",
"position",
"sentenceText",
"numSentencesInBook",
# count_sentence_multisyllables_udf('sentenceText') \
# .alias("multiSyllableCount")
count_array_multisyllables_udf("words") \
.alias("multiSyllableCount"),
func.size("words").alias("numWordsInSentence"),
)
sentences.printSchema()
# pipeline = spark_nlp.setup_sentiment_pipeline()
# output = spark_nlp.sentiment_analysis(sentence_data, pipeline)
# Format to load ElasticSearch
sentences = sentences.select(
"sentence_id",
func.to_json(
func.struct("sentence_id", "fileName", "position", "sentenceText",
"numSentencesInBook", "multiSyllableCount",
"numWordsInSentence")).alias("value"))
# sentence = output.select(["sentence_id", "sentence"]).toJSON()
sentences = sentences.rdd.map(lambda x: elastic.format_data(x))
# Write to ES
write_rdd_to_es(sentences, es_write_conf)
"""
# _read_es()
# sentences = sentence.rdd.map(lambda s: s.sentence[0].result)
# sentences = sentence.rdd.flatMap(lambda s: s.sentence)
# results = sentence.rdd.map(lambda s: s.result).zipWithUniqueId()
"""
spark_session.stop()
end_time = time.time()
logging.info("RUNTIME: {0}".format(end_time - start_time))
def parse_arguments(self): self.args = utils.parse_arguments()
def run():
args = utils.parse_arguments()
logger.info('Hitting main...')
main(args)
def handle_command(slack_client, sender, command, channel):
"""
Executes bot command if the command is known
"""
# Default response is help text for the user
global em
default_response = "Sorry .. *Sergio* and *Taeyeong* haven't taught me how to respond to your command .. "
# Finds and executes the given command, filling in response
response = ""
# get info about users in channel
users_list = slack_client.api_call("users.list", token=AUTH_TOKEN)
members = users_list["members"]
all_ids = [member["id"] for member in members] # extract only names
all_names = [member["real_name"]
for member in members] # extract only names
sender_email = [
member["profile"]["email"] for member in members
if member['id'] == sender
][0]
sender_name = [
member["real_name"] for member in members if member['id'] == sender
][0]
# This is where you start to implement more commands!
if command.startswith(CREATE_COMMAND):
#### error handling needed for invalid guests
em, success = update_event(em,
sender,
command,
all_ids,
all_names,
user_email=sender_email,
user_name=sender_name,
create=True)
event = em.dict[sender]
# message
response += "Creating this event:"
response += '\n' + event.export_str()
response += "\n\nIf you want to add more info, please use `{}` command, or if you want to push to your Google Calendar account, use `{}` command.".format(
ADD_COMMAND, PUSH_COMMAND)
elif command.startswith(ADD_COMMAND):
event = em.find_event(sender)
if event == None:
response += "You haven't created any event yet. Use `{}` command first!".format(
CREATE_COMMAND)
else:
users_list = slack_client.api_call("users.list", token=AUTH_TOKEN)
members = users_list["members"]
all_ids = [member["id"]
for member in members] # extract only names
all_names = [member["name"]
for member in members] # extract only names
em, success = update_event(em,
sender,
command,
all_ids,
all_names,
create=False)
event = em.find_event(sender)
response += '\n' + event.export_str()
response += "\nIf you want to add more info, please use `{}` command.\nIf you want to push to your Google Calendar account, use `{}` command".format(
ADD_COMMAND, PUSH_COMMAND)
elif command.startswith(MODIFY_COMMAND):
items = command.split()
if len(items) < 4:
response += "You must enter `{}` command, event ID, field to change, and its new value".format(
MODIFY_COMMAND)
else:
arg_to_change = parse_arguments(command)
em, success = update_event(em,
sender,
command,
all_ids,
all_names,
user_email=sender_email,
user_name=sender_name,
create=True)
editEvent(google_service, sender_email, items[1], arg_to_change,
em.find_event(sender))
response += "Event updated."
elif command.startswith(PUSH_COMMAND):
event = em.find_event(sender)
if event == None:
response += "You haven't created any event yet. Use `{}` command first!".format(
CREATE_COMMAND)
else:
event_id = createEvent(google_service, event.title, event.loc,
event.desc, event.time, event.endtime,
event.host_email, event.guest_emails,
event.alert)
em.delete_event(sender)
response += "Your event has been created! The event ID is *{}* you can use to `{}` or `{}`".format(
event_id, MODIFY_COMMAND, DELETE_COMMAND)
response += "\nNow, you don't have any event to `{}`.".format(
PUSH_COMMAND)
elif command.startswith(LIST_COMMAND):
event = em.find_event(sender)
if event == None:
response += "You haven't created any event yet. Use `{}` command first!".format(
CREATE_COMMAND)
else:
response += '\n' + event.export_str()
response += "\n\nIf you want to add more info, please use `{}` command, or if you want to push to Google Calendar, use `{}` command.".format(
ADD_COMMAND, PUSH_COMMAND)
elif command.startswith(INIT_COMMAND):
em.delete_event(sender)
response += "Now, you don't have any event to `{}`.".format(
PUSH_COMMAND)
elif command.startswith(HELP_COMMAND):
response += show_help()
elif command.startswith(DELETE_COMMAND):
items = command.split()
if len(items) != 2:
response += "You must enter `{}` command and 1 event ID.".format(
DELETE_COMMAND)
else:
event_id = items[1]
deleteEvent(google_service, sender_email, event_id)
response += "Event deleted."
# Sends the response back to the channel
slack_client.api_call("chat.postMessage",
channel=channel,
text=response or default_response)
else:
conn.sendall(make_message('unknowncommand'))
conn.close()
logger.info('Close thread connection...')
def run(self):
logger.info('Run server... HOST={} PORT={}'.format(self.host, self.port))
self.socket.listen(self.MAX_CONNECTIONS)
while not self.is_stop:
conn, addr = self.socket.accept()
logger.info('Connect to {}'.format(addr))
self.create_thread(conn, addr)
print(111)
self.shutdown()
def shutdown(self):
for thread in self.threads:
thread.join()
self.socket.close()
logger.info('Server shutdown.')
if __name__ == '__main__':
args = parse_arguments()
server = Server(args.host, args.port)
server.run()
)
)
mutual_exclusive.add_argument(
'--brian2cuda',
action='store_true',
help=(
"Pass ``test`` string as ``-k`` option to select tests from brian2cuda test "
"suite."
)
)
parser.add_argument('--only',
choices=['single-run', 'multi-run', 'standalone-only'],
default=None)
args = utils.parse_arguments(parser)
import sys, os, pytest
from io import StringIO
import numpy as np
import brian2
import brian2cuda
from brian2.devices.device import reset_device
from brian2.tests import clear_caches, make_argv, PreferencePlugin
from brian2 import prefs
import brian2cuda
all_prefs_combinations = utils.set_preferences(args, prefs)
# target independent preferences
def __init__(self):
self.args = utils.parse_arguments()
self.ipc_fuzzer_binary = utils.get_fuzzer_application_name()
self.ipc_fuzzer_binary_path = utils.get_application_path(
self.ipc_fuzzer_binary)