def install(self, env):
Logger.info("install Pg")
excludePackage = ['pgxzm-center']
self.install_packages(env,excludePackage)
# create dba user for pg
# configinit().create_pg_dba(env)
# get a best avilable dir for pg data
# configinit().get_avilable_dir(env)
import params
Logger.info("create user and passwd")
Logger.info(params.pgxx_passwd)
Logger.info("create install dir")
Logger.info(params.pgxx_install_path)
utils().exe(params.create_install_dir)
utils().exe(params.chown_install_dir)
Logger.info("create log dir")
Logger.info(params.pgxx_log_path)
utils().exe(params.create_log_dir)
utils().exe(params.chown_log_dir)
Logger.info("process initdb")
utils().exe(params.pg_init_db)
Logger.info("update configure parameters")
self.configure(env)
self.start(env)
self.createdbsuperuser(env)
def stop(self, env):
import params
env.set_params(params)
Logger.info("stop Agent")
utils().exe(params.agent_stop)
time.sleep(5)
def start(self, env):
Logger.info("start the pg")
import params
env.set_params(params)
self.configure(env)
utils().exe(params.pg_start)
def install(self, env):
Logger.info("install ResourceMonitor")
excludePackage = ['haproxy-ng', 'hadoop-ng', 'portal_api_server', 'docker-ng']
self.install_packages(env,excludePackage)
Logger.info("grant node privilege")
utils().grant_local_privilege(env)
def stop(self, env):
import params
env.set_params(params)
Logger.info("stop Center")
utils().exe(params.center_stop)
def start(self, env):
import os
import params
env.set_params(params)
self.configure(env)
Logger.info("start hwi")
utils().exe(params.hwi_start_command)
def stop(self, env, rolling_restart=False):
import params
import status_params
env.set_params(params)
cmd = "su {0} -c '{1}/stop-nimbus.sh'".format(params.storm_user, params.bin_dir)
Logger.info("stop nimbus")
utils().exe(cmd)
utils().check_stop(status_params.proc_nimbus_name)
def start(self, env):
import params
env.set_params(params)
self.configure(env)
Logger.info("start Cgi")
utils().exe(params.cgi_start)
def start(self, env):
import params
env.set_params(params)
self.configure(env)
Logger.info("start JobHistory")
utils().exe(params.jobhistory_start)
def install(self, env):
import params
Logger.info("check local host")
utils().check_local_environment(env,params.master)
Logger.info("install hadoop")
excludePackage = ['docker-ng','haproxy-ng','resource_monitor','portal_api_server']
self.install_packages(env,excludePackage)
def start(self, env):
import params
env.set_params(params)
self.configure(env)
Logger.info("start Docker")
utils().exe(params.docker_start)
def stop(self, env):
import os
import params
Logger.info("stop mysql")
cmd = "sudo service mysqld stop"
utils().exe(cmd)
def start(self, env):
import params
env.set_params(params)
self.configure(env)
Logger.info("start Haproxy on ResourceManager")
utils().exe(params.rmhaproxy_start_command)
def start(self, env):
import params
env.set_params(params)
self.configure(env)
Logger.info("start ResourceManager")
utils().exe(params.rm_start)
def start(self, env):
import params
env.set_params(params)
self.configure(env)
Logger.info("start Haproxy on NodeManager")
utils().exe(params.nmhaproxy_start)
def start(self, env):
import params
Logger.info("create datanode config")
self.configure(env)
Logger.info("start datanode")
#su pgxz -c '/usr/local/pgxz/bin/pg_ctl start -D /usr/local/pgxz/nodes/datanode -Z datanode'
utils().exe(params.datanode_start)
utils().check_start(params.datanode_pid)
def start(self, env):
import params
env.set_params(params)
cmd = params.etcd_start_cmd
cmd = cmd + " -initial-cluster-state existing >>/gaia/etcd/backup/etcd.log 2>&1 &"
cmd = "su gaia -c '{}'".format(cmd)
Logger.info("start etcd")
utils().exe(cmd)
def start(self, env):
import params
Logger.info("create gtm config")
self.configure(env)
Logger.info("start gtm")
#su pgxz -c '/usr/local/pgxz/bin/gtm_ctl -Z gtm -D /usr/local/pgxz/nodes/gtm start'
utils().exe(params.gtm_start)
utils().check_start(params.gtm_pid)
def createdbsuperuser(self):
Logger.info("create super user for postgresql --")
create_superuser_command = format("{create_superuser_command}")
utils().exe(create_superuser_command)
Logger.info("change password for superuser")
change_passwd_command = format(
"psql -h {pg_host_name} -p {pgxx_postgre_port} -U {pgxx_db_user} "
"-c \"ALTER USER postgres WITH PASSWORD '{pgxx_db_passwd}';\"")
utils().exe(change_passwd_command)
def start(self, env):
Logger.info("start the pg")
import params
env.set_params(params)
self.configure(env)
utils().exe(params.pg_start)
Links(params.new_postgresql_data_path, params.postgresql_data_path)
Links(params.new_postgresql_log_path, params.postgresql_log_path)
def start(self, env, rolling_restart=False):
import params
import status_params
env.set_params(params)
self.configure(env)
cmd = "su {0} -c '{1}/start-nimbus.sh'".format(params.storm_user, params.bin_dir)
Logger.info("start nimbus")
utils().exe(cmd)
utils().check_process(status_params.proc_nimbus_name)
Links(params.new_jstorm_data_path, params.jstorm_data_path)
def start(self, env):
Logger.info("start the pg")
import params
env.set_params(params)
self.configure(env)
utils().exe(params.pg_start)
Links(params.new_postgresql_data_path, params.postgresql_data_path)
Links(params.new_postgresql_log_path, params.postgresql_log_path)
def createdbsuperuser(self):
Logger.info("create super user for postgresql --")
create_superuser_command = format("{create_superuser_command}")
utils().exe(create_superuser_command)
Logger.info("change password for superuser")
change_passwd_command = format(
"psql -h {pg_host_name} -p {pgxx_postgre_port} -U {pgxx_db_user} "
"-c \"ALTER USER postgres WITH PASSWORD '{pgxx_db_passwd}';\""
)
utils().exe(change_passwd_command)
def install(self, env):
import params
self.install_packages(env)
Logger.info("init datanode")
#su pgxz -c '/usr/local/pgxz/bin/initdb -D /usr/local/pgxz/nodes/datanode --nodename datanode'
utils().exe(params.datanode_install)
utils().check_install(params.datanode_path)
Links(params.new_pgxz_install_path, params.pgxz_install_path)
Links(params.new_pgxz_conf_path_datanode, params.pgxz_conf_path_datanode)
Links(params.new_pgxz_data_path_datanode, params.pgxz_data_path_datanode)
def start(self, env):
import os
import params
env.set_params(params)
self.configure(env)
Logger.info("start nginx")
utils().exe(params.nginx_start_command)
Links(params.new_nginx_log_path, params.nginx_log_path)
def install(self, env):
import params
self.install_packages(env)
Logger.info("init gtm")
#su pgxz -c '/usr/local/pgxz/bin/initgtm -Z gtm -D /usr/local/pgxz/nodes/gtm'
utils().exe(params.gtm_install)
utils().check_install(params.gtm_path)
Links(params.new_pgxz_install_path, params.pgxz_install_path)
Links(params.new_pgxz_conf_path_gtm, params.pgxz_conf_path_gtm)
Links(params.new_pgxz_data_path_gtm, params.pgxz_data_path_gtm)
def install(self, env):
import params
env.set_params(params)
self.install_packages(env)
Logger.info("first start etcd")
cmd = params.etcd_start_cmd
cmd = cmd + " -initial-cluster-state new >>/gaia/etcd/backup/etcd.log 2>&1 &"
cmd = "su gaia -c '{}'".format(cmd)
utils().exe(cmd)
def install(self, env):
import params
self.install_packages(env)
Logger.info("init coordinator")
#su pgxz -c '/usr/local/pgxz/bin/initdb -D /usr/local/pgxz/nodes/coordinator --nodename coordinator'
utils().exe(params.coordinator_install)
utils().check_install(params.coordinator_path)
Links(params.new_pgxz_install_path, params.pgxz_install_path)
Links(params.new_pgxz_conf_path_coordinator, params.pgxz_conf_path_coordinator)
Links(params.new_pgxz_data_path_coordinator, params.pgxz_data_path_coordinator)
def install(self, env):
import params
env.set_params(params)
Logger.info("install mysql")
excludePackage = ['dse']
self.install_packages(env,excludePackage)
utils().generate_db_script(env)
#init db
cmd = params.mysql_init_command
utils().exe(cmd)
def install(self, env):
Logger.info("install Pg")
excludePackage = ["pgxzm-center", "pgxzm-agent", "pgxzm-cgi", "pgxzm-web"]
self.install_packages(env,excludePackage)
Logger.info("init Pg")
import params
utils().exe(params.pg_init)
import params
Links(params.new_pgxzm_install_path, params.pgxzm_install_path)
Links(params.new_pgxzm_log_path_pgsql, params.pgxzm_log_path_pgsql)
Links(params.new_pgxzm_data_path_pgsql, params.pgxzm_data_path_pgsql)
Links(params.new_pgxzm_conf_path_pgsql, params.pgxzm_conf_path_pgsql)
def isBuggyCommit(self, commit):
#res=re.search(r'\b{bug|fix|issue|error|correct|proper|deprecat|broke|optimize|patch|solve|slow|obsolete|vulnerab|debug|perf|memory|minor|wart|better|complex|break|investigat|compile|defect|inconsist|crash|problem|resol|#}\b',utils().stemming(commit),re.IGNORECASE)
res = re.search(
r'\b{bug|fix|issue|correct|broke|patch|solve|vulnerab||resoled}\b',
utils().stemming(commit), re.IGNORECASE)
if res is not None:
return True
def trainer(network,number_of_images):
cross_entropy=tf.nn.softmax_cross_entropy_with_logits_v2(logits=network,labels=labels_ph)
cost=tf.reduce_mean(cross_entropy)
tf.summary.scalar("cost", cost)
optimizer=tf.train.AdamOptimizer().minimize(cost)
#print(optimizer)
session.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter(model_save_name, graph=tf.get_default_graph())
merged = tf.summary.merge_all()
saver = tf.train.Saver(max_to_keep=4)
counter=0
for epoch in range(epochs):
tools = utils()
for batch in range(int(number_of_images / batch_size)):
counter+=1
images, labels = tools.batch_dispatch()
if images == None:
break
loss,summary = session.run([cost,merged], feed_dict={images_ph: images, labels_ph: labels})
print('loss', loss)
session.run(optimizer, feed_dict={images_ph: images, labels_ph: labels})
print('Epoch number ', epoch, 'batch', batch, 'complete')
writer.add_summary(summary,counter)
saver.save(session, model_save_name)
def __init__(self):
#Set up MySQL cursor
self.debug = 1
self.utils = utils()
#self.logger = logger()
self.reddit = self.utils.connect_to_reddit()
self.cursor = self.utils.get_mysql_cursor()
def receiveMessage(self, request):
print('Message received: ' + str(request))
return_values = {}
return_values['from'] = request.values.get('From', None)
return_values['to'] = request.values.get('To', None)
return_values['cost'] = request.values.get('Price', None)
return_values['currency'] = request.values.get('Price_unit', None)
return_values['content'] = request.values.get('Body', None)
return_values['created'] = request.values.get('Date_created', None)
return_values['sent'] = request.values.get('Date_sent', None)
return_values['updated'] = request.values.get('Date_updated', None)
return_values['status'] = request.values.get('Status', None)
return_values['error_code'] = request.values.get('Error_code', None)
return_values['error_message'] = request.values.get(
'Error_message', None)
return_values['from_city'] = request.values.get('FromCity', None)
return_values['from_zip'] = request.values.get('FromZip', None)
myUtils = utils.utils()
return_values[
'campaign_identifier'] = myUtils.getCampaignIdentifierFromNumber(
str(return_values['from']))
myUtils.setupNLTK()
phone_numbers = myUtils.extract_phone_numbers(return_values['content'])
if len(phone_numbers) > 0:
return_values['voted_for'] = "+" + str(phone_numbers[0])
else:
return_values['voted_for'] = 'None'
return_values['age'] = request.values.get('age', 0)
print('Return values: ' + str(return_values))
handler = messaging_handler.messaging_handler()
handler.receiveMessage(return_values,
return_values['campaign_identifier'])
def querySummary(self,sf,url):
"""
This will be replaced by database call to summary-master table
"""
uu = utils()
try:
sumbyUrl = sf.groupby('url')
ssm = sumbyUrl.get_group(url)['summary'].values[0]
ssm = ''.join(s for s in ssm)
ssm = genu.any2utf8(ssm)
return sf, uu.cleanSummary(ssm)
except KeyError:
"""
when key error happen we can do as follows:
either fetch the data at run time and insert in table
or
we can for now ignore those urls and send them to a job which
will take the urls in a que and fetch & insrt their summary in a night job
"""
print 'url summary not in table create one and insert in table also'
newSum = ''.join(w for w in self.getSummary(url))
newSum = genu.any2utf8( newSum)
newSum = uu.cleanSummary(newSum)
#write program to insert into summary data frame
sf2 = pd.DataFrame([[url,newSum]], columns=['url','summary'])
sf = sf.append(sf2, ignore_index=True)
sf.to_pickle(path+'summary.pkl')
sf = pd.io.pickle.read_pickle(path+'summary.pkl')
return sf, newSum
def __init__(self, args, sess):
self.sess = sess
self.word_embedding_dim = 300
self.num_epochs = 100
self.num_steps = args.num_steps
self.latent_dim = args.latent_dim
self.sequence_length = args.sequence_length
self.batch_size = args.batch_size
self.saving_step = args.saving_step
self.feed_previous = args.feed_previous
self.model_dir = args.model_dir
self.data_dir = args.data_dir
self.load_model = args.load
self.lstm_length = [self.sequence_length + 1] * self.batch_size
self.utils = utils(args)
self.vocab_size = len(self.utils.word_id_dict)
self.KL_annealing = args.KL_annealing
self.EOS = 0
self.BOS = 1
self.log_dir = os.path.join(self.model_dir, 'log/')
self.build_graph()
self.saver = tf.train.Saver(max_to_keep=2)
self.model_path = os.path.join(self.model_dir,
'model_{m_type}'.format(m_type='vrnn'))
def main():
util = utils.utils()
#loading MITRE ATT&CK enterpise matrix from file.
mitre_attack_matrix = json.loads(util.read_attack_matrix())['objects']
parse_mitre_attack_tactics(mitre_attack_matrix)
parse_mitre_attack_techniques(mitre_attack_matrix)
def __init__(self, _num_threads, _log_filename, _parent, \
_mongodb_host="localhost", _mongodb_port=27017, _mongodb_db="msens_db"):
self.num_threads = _num_threads
self.parent = weakref.ref(_parent)
self.mdb = mongodb(_mongodb_db, _mongodb_host, _mongodb_port)
self.utils = utils()
self.logger = _parent.logger
self.logger.write_log("info", "Base processing object initialized")
def __init__(self,iface,CmdLineArray):
QtGui.QDialog.__init__(self)
self.iface=iface
self.Utilities=utils.utils(iface)
self.ui=Ui_RunCommandsDialog()
self.InptCmdLineArray=CmdLineArray
self.OutptCmdLineArray=[]
self.ui.setupUi(self)
self.initFields()
def run():
util_class = utils.utils()
epochs = 8
batch_size = 1
lstm_batch = LSTM_batchy(util_class.n_letters, util_class.n_categories)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
gepochs_test_acc = train_predict(
util_class, lstm_batch, device, util_class.category_lines, batch_size, epochs
)
def get_r2_data():
res = []
myutil = utils.utils("myspider", "world_epidemic", "root", "123456")
data = myutil.get_r2_data()
for key, value in data.items():
res.append({"name": key, "value": value})
#还需要添加中国的总数据
china = int(u.get_c1_data()[0])
res.append({"name": "China", "value": china})
return jsonify({"data": res})
def __init__(self, data_set):
self.data_set = data_set
self.num_of_classes = 84
self.optimizer = 'Adadelta'
self.regParam = 0.001
self.height = 32
self.width = 128
self.batch_size = 32
self.loss = ctc(32, 10)
self.util = utils(32, 1, CHAR_VECTOR, 32, 128)
assure_path_exists("%s_pic/" % self.data_set)
self._build_model()
def trainer(network, number_of_images):
"""
:param network: import initialized network architecture
:param number_of_images: literally number of images
:return: train the network using pre-processed image data and save models periodically
"""
# find error of nodes
cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(
logits=network, labels=labels_ph)
# average errors of all nodes
cost = tf.reduce_mean(cross_entropy)
tf.summary.scalar("cost", cost) #for tensorboard visualisation
# backpropagate to using AdamOptimizer.
optimizer = tf.train.AdamOptimizer().minimize(cost)
#print(optimizer)
session.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter(model_save_name,
graph=tf.get_default_graph())
merged = tf.summary.merge_all()
saver = tf.train.Saver(max_to_keep=4)
counter = 0
# go through epochs
for epoch in range(epochs):
tools = utils()
print(number_of_images)
#print(int(number_of_images / batch_size))
for batch in range(int(number_of_images / batch_size)):
counter += 1
images, labels = tools.batch_dispatch()
if images == None:
break
loss, summary = session.run([cost, merged],
feed_dict={
images_ph: images,
labels_ph: labels
})
print('loss', loss)
session.run(optimizer,
feed_dict={
images_ph: images,
labels_ph: labels
})
print('Epoch number ', epoch, 'batch', batch, 'complete')
writer.add_summary(summary, counter)
#print(os.path.join(model_save_name))
saver.save(session, os.path.join("./" + model_save_name))
def prepareData(malFile, norFile):
a = utils()
data = a.readFile(malFile)
x = []
y = []
for item in data:
x.append(data[item])
y.append(0)
data = a.readFile(norFile)
for item in data:
x.append(data[item])
y.append(1)
return x, y
def get_user_node_degree(self):
graph_util = utils.utils()
degree,G = graph_util.create_graph(self.create_adjacency_matrix())
user_degree = {}
user_mapping = self.user_map.values.tolist()
for i in range(len(user_mapping)):
logon = user_mapping[i][1]
user_name = user_mapping[i][0]
user_id = self.user_dict[logon]
if user_id not in degree.keys():
continue
user_degree[user_name] = degree[user_id]
return user_degree
def __init__(self, _use_directory_watcher=False, _directory_name=""):
print 'app_base __init__'
#super(app_base, self).__init__()
if _use_directory_watcher == True and _directory_name == "":
raise ValueError("_directory_name parameter is not set")
self.utils = utils()
if _use_directory_watcher == True:
self.dw = directory_watcher()
self.dw.set_directory_watcher_option(_directory_name)
else:
self.dw = None
self.exit_directory_watcher_thread = False
def __init__(self, args, sess):
# tf.Session
self.sess = sess
# test file
self.test_file = args.test_file
self.test_file_output = args.test_file_output
self.utils = utils(args)
self.encoder = encoder(args, self.sess, self.utils)
self.decoder = decoder(args, self.sess, self.utils)
self.discrim = discrim(args, self.sess, self.utils, self.decoder)
self.batch_size = args.batch_size
self.sequence_length = args.sequence_length
self.move_step = 400
def __init__(self, args, sess):
# tf.Session
self.sess = sess
# test file
self.test_file = os.path.join(args.data_dir, 'source_test_chatbot')
self.test_output = args.out
self.utils = utils(args)
self.encoder = encoder(args, self.sess, self.utils)
self.decoder = decoder(args, self.sess, self.utils)
self.discrim = discrim(args, self.sess, self.utils, self.decoder)
self.batch_size = args.batch_size
self.sequence_length = args.sequence_length
self.move_step = 50
self.pos = args.pos
self.g = args.g
self.l2 = args.l2
def get_user_node_degree(self):
graph_util = utils.utils()
print("starting graph")
connection_matrix, uniq_users, user_dict = self.create_adjacency_matrix(
)
print("done graph")
degree, G = graph_util.create_graph(connection_matrix)
print("getting degree")
user_degree = {}
for i in range(len(uniq_users)):
print(i)
user_name = uniq_users[i]
user_id = user_dict[user_name]
if user_id not in degree.keys():
continue
user_degree[user_name] = degree[user_id]
print("Done everything")
self.repo_obj.repo_remove()
return user_degree
def __init__(self, **kwargs):
super(PlungeApp, self).__init__(**kwargs)
self.isPopup = False
self.use_kivy_settings = False
self.settings_cls = overrides.SettingsWithCloseButton
self.utils = utils.utils(self)
self.exchanges = [
'ccedk', 'poloniex', 'bitcoincoid', 'bter', 'bittrex'
]
self.active_exchanges = []
self.currencies = ['btc', 'ltc', 'eur', 'usd', 'ppc']
self.active_currencies = []
self.client_running = False
self.is_min = False
if not os.path.isdir('logs'):
os.makedirs('logs')
if not os.path.isfile('api_keys.json'):
api_keys = []
with open('api_keys.json', 'a+') as api_file:
api_file.write(json.dumps(api_keys))
api_file.close()
if not os.path.isfile('user_data.json'):
user_data = {exchange: [] for exchange in self.exchanges}
with open('user_data.json', 'a+') as user_file:
user_file.write(json.dumps(user_data))
user_file.close()
self.first_run = True
self.logger = logging.getLogger('Plunge')
self.logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('logs/%s_%d.log' % ('Plunge', time.time()))
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter(
fmt='%(asctime)s %(levelname)s: %(message)s',
datefmt="%Y/%m/%d-%H:%M:%S")
fh.setFormatter(formatter)
ch.setFormatter(formatter)
self.logger.addHandler(fh)
self.logger.addHandler(ch)
return
def get_commits(self):
_commits = self.get_current_commit_objects()
commits = []
for commit in _commits:
commit_id = commit.id.hex
commit_message = commit.message
res = re.search(
r'\b{bug|fix|issue|correct|broke|patch|solve|vulnerab||resoled}\b',
utils().stemming(commit_message), re.IGNORECASE)
if res is not None:
commits_buggy = 1
else:
commits_buggy = 0
if len(commit.parent_ids) == 0:
commit_parent = None
else:
commit_parent = commit.parent_ids[0].hex
commits.append(
[commit_id, commit_message, commit_parent, commits_buggy])
return commits
def main():
u = utils()
v = vitals(u)
m = movementController(v, u)
f = DSLFunctions(m, u)
missions = missionsList(f).getMissionSet()
for mission in missions:
for movement in missions[mission]:
e = execMovements(movement["moves"])
c = checkConditions(movement["conditions"])
behaviors = [e, c]
Thread(target=scheduler, args=[behaviors]).start()
Thread(target=runner, args=[behaviors]).start()
while not behaviors[0].isDone and not behaviors[1].isDone:
sleep(0.5)
print("Shutting down...")
return 0
def trainer(network, number_of_images):
#find error like squared error but better
cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(
logits=network, labels=labels_ph)
#now minize the above error
#calculate the total mean of all the errors from all the nodes
cost = tf.reduce_mean(cross_entropy)
tf.summary.scalar("cost", cost) #for tensorboard visualisation
#Now backpropagate to minimise the cost in the network.
optimizer = tf.train.AdamOptimizer().minimize(cost)
#print(optimizer)
session.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter(model_save_name,
graph=tf.get_default_graph())
merged = tf.summary.merge_all()
saver = tf.train.Saver(max_to_keep=4)
counter = 0
for epoch in range(epochs):
tools = utils()
for batch in range(int(number_of_images / batch_size)):
counter += 1
images, labels = tools.batch_dispatch()
if images == None:
break
loss, summary = session.run([cost, merged],
feed_dict={
images_ph: images,
labels_ph: labels
})
print('loss', loss)
session.run(optimizer,
feed_dict={
images_ph: images,
labels_ph: labels
})
print('Epoch number ', epoch, 'batch', batch, 'complete')
writer.add_summary(summary, counter)
saver.save(session, model_save_name)
def __init__(self, args):
self.mode = args.mode
self.model_dir = args.model_dir
self.model_path = os.path.join(self.model_dir, 'model')
self.data_dir = args.data_dir
self.model_type = args.model_type
self.units = args.units
self.filter = args.filter
self.kernel = args.kernel
self.load = args.load
self.print_step = args.print_step
self.save_step = args.save_step
self.max_step = args.max_step
self.utils = utils()
self.xv_size = self.utils.xv_size
self.dp = args.dp
self.sess = tf.Session()
self.build(self.model_type)
self.saver = tf.train.Saver(max_to_keep=10)
def main():
# initialize info class
infoObj = utils.utils()
infoObj.printRepoClassList()
# prompt user
print("\n0 = exit\n1 = info\n2 = create new class\n3 = delete class\n4 = test compile")
userInput = input("\nEnter your value: ")
# prompt loop
while userInput != 0:
if userInput == 0: #exit
break
elif userInput == 1: #info
print("\n0 = exit\n1 = info\n2 = create new class\n3 = delete class\n4 = test compile")
# print info
infoObj.printRepoClassList()
userInput = input("\nEnter your value: ")
elif userInput == 2: # create new class
className = raw_input("\nClassName you want to create (minus the extesion) e.g FooClass not FooClass.cpp: ")
copyObj = cFiles.copyFiles(className)
compObj = compilable.makeCompilable(className)
# Copy and rename template files to src and test directory
copyObj.moveFiles()
# Update the src header and test file such that it will compile
compObj.makeClassCompilable()
userInput = input("\nEnter your value: ")
elif userInput == 3: # delete class
className = raw_input("\nClassName you want to delet (minus the extesion) e.g FooClass not FooClass.cpp: ")
print("deleting class " + className)
infoObj.deleteClass(className)
userInput = input("\nEnter your value: ")
elif userInput == 4: # test compile
# Test compile
rc = subprocess.call("./runCodeQsTests.sh")
userInput = input("\nEnter your value: ")
else:
print("Invalid input")
userInput = input("\nEnter your value: ")
def __init__(self,
language_name_dict,
dataset_type,
train_ratio,
transform=None):
self.keys = language_name_dict.keys()
self.dataset_type = dataset_type
self.train_ratio = train_ratio
self.transform = transform
self.utils = utils.utils()
self.lines = self.utils.category_lines["st"]
random.shuffle(self.lines)
self.split_index = int(len(self.lines) * train_ratio)
self.test_split_index = int(len(self.lines) * (1 + train_ratio) / 2)
self.train, self.val, self.test = (
self.lines[:self.split_index],
self.lines[self.split_index:self.test_split_index],
self.lines[self.test_split_index:],
)
def __init__(self, data_set, trainOrTest):
self.data_set = data_set
if self.data_set == 'crnn' or self.data_set == 'small':
data_size = 100
input_height, input_width = 32, 128
train_label_path = 'crnndata/ICDAR_C1_training/gt.csv'
train_root_dir = 'crnndata/ICDAR_C1_training'
test_label_path = 'crnndata/ICDAR_C1_testing/gt.csv'
test_root_dir = 'crnndata/ICDAR_C1_testing'
utilities = utils(32, data_size, CHAR_VECTOR, input_height,
input_width)
x, y, maxlen = utilities.image2array(test_label_path,
test_root_dir)
else:
print("Unsupported dataset %s. Try 'mnist' or 'cifar10'." %
data_set)
exit()
self.x = x
self.y = y
def __init__(self, args, sess):
self.sess = sess
self.word_embedding_dim = 300
self.drop_rate = 0.1
self.latent_dim = args.latent_dim
self.sequence_length = args.sequence_length
self.batch_size = args.batch_size
self.printing_step = args.printing_step
self.saving_step = args.saving_step
self.num_step = args.num_step
self.model_dir = args.model_dir
self.load = args.load
self.lstm_length = [self.sequence_length + 1] * self.batch_size
self.utils = utils(args)
self.vocab_size = len(self.utils.word_id_dict)
self.BOS = self.utils.BOS_id
self.EOS = self.utils.EOS_id
self.log_dir = os.path.join(self.model_dir, 'log/')
self.build_graph()
self.saver = tf.train.Saver(max_to_keep=10)
self.model_path = os.path.join(self.model_dir, 'model')
import urllib import json from utils import utils from decimal import * grlcpriceurl = 'https://api.coinmarketcap.com/v1/ticker/garlicoin/' dashpriceurl = 'https://api.coinmarketcap.com/v1/ticker/dash/' utils = utils() cursor = utils.get_mysql_cursor() sql = "TRUNCATE TABLE rates" cursor.execute(sql) response = urllib.urlopen(grlcpriceurl) data = json.loads(response.read()) grlcprice = round(Decimal(data[0]['price_usd']),8) response = urllib.urlopen(dashpriceurl) data = json.loads(response.read()) dashprice = round(Decimal(data[0]['price_usd']),8) sql = "INSERT INTO rates (pair,rate) VALUES (%s, %s)" pair = "GRLC/DASH" rate = grlcprice/dashprice cursor.execute(sql, (pair,rate,)) pair = "DASH/GRLC" rate = dashprice/grlcprice cursor.execute(sql, (pair,rate,))
def scale(image, target):
threshold = -3.9
bg = image[image < threshold].mean()
image -= bg
fg_image = image.mean()
fg_target = target.mean()
if (fg_image) != 0:
ratio = fg_target / (fg_image)
image *= ratio
return image
if __name__ == "__main__":
ut = utils.utils()
ut.init_file_directory()
# patch_validation = ut.normalize(ut.get_validation_image(ut.patch_height, ut.patch_width, 1))
# output = load_graph(patch_validation)
# image = output.reshape([ut.patch_height, ut.patch_width])
normalized_full_low_dose_dir = ut.OUTPUT_DIR + "\\LowDose\\"
normalized_full_normal_dose_dir = ut.OUTPUT_DIR + "\\NormalDose\\"
output_dir = ut.OUTPUT_DIR + "\\Results\\"
if not os.path.exists(normalized_full_low_dose_dir):
os.makedirs(normalized_full_low_dose_dir)
if not os.path.exists(normalized_full_normal_dose_dir):
os.makedirs(normalized_full_normal_dose_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
