def image_preprocessing(show_result=False):
if re_split:
split.split()
# initial undistort map
map1, map2 = init_undistort_image(split.parameters)
for i in range(split.parameters['NumPatterns']):
print("preprocessing image No.{} ...".format(i))
# open image file
folder_name = "camera-calibration"
image_origin = cv2.imread(folder_name + "/{}.jpg".format(i))
image = cv2.imread(folder_name + "/{}_result.jpg".format(i))
# undistorting
undistorted_image = cv2.remap(image, map1, map2, cv2.INTER_LINEAR)
undistorted_image_origin = cv2.remap(image_origin, map1, map2,
cv2.INTER_LINEAR)
cv2.imwrite(folder_name + "/{}_undistorted.jpg".format(i),
undistorted_image_origin)
cv2.imwrite(folder_name + "/{}_result_undistorted.jpg".format(i),
undistorted_image)
if show_result:
cv2.namedWindow("input", cv2.WINDOW_NORMAL)
cv2.imshow("input", image)
cv2.namedWindow("origin", cv2.WINDOW_NORMAL)
cv2.imshow("origin", image_origin)
cv2.namedWindow("undistorted image", cv2.WINDOW_NORMAL)
cv2.imshow("undistorted image", undistorted_image)
cv2.waitKey(0)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--splitScoreOnly", action="store_true",
help="Split recorded files only"
)
parser.add_argument("scoreFilename", nargs="?" ,
help="Input score filename",
default=settings.defaultScoreFilename
)
parser.add_argument("outputDir", nargs="?" ,
help="Expressive output directory",
default=settings.defaultOutputDir
)
args = parser.parse_args()
if settings.DEBUG: print("[DEBUG] Filename: " + args.scoreFilename)
try:
score = music21.converter.parse(args.scoreFilename)
except music21.converter.ConverterException:
print("[WARN] " + args.scoreFilename+ " is not a score file, skipped.")
raise
#recLen = 0;
#scoreLen = len(score.flat.notes)
if settings.DEBUG:
print("[DEBUG] "),
score.flat.show('text')
#recLogFilename = settings.getRecLogFilename(args.scoreFilename)
notes = score.flat.notes
if (not args.splitScoreOnly):
recordAll(notes, args = args)
split.split(settings.getSplitRecFilename(args.scoreFilename, args.outputDir),
args.scoreFilename, args.outputDir)
def main():
warnings.warn = warn
mk_dir()
print('# split\n')
split()
print('# building dictionary\n')
for file in tqdm(os.listdir('TEMP')):
statinfo = os.stat('TEMP/'+file)
#check if it is empty shell
if statinfo.st_size>1000:
try:
df = build_dict(file)
df = agg(df)
#cant use if df on it
if len(df.index)!=0:
df['size'] = df['size'] = statinfo.st_size
df['raw_label'] = file.replace('.pcap','')
df.to_csv('TEMP_CSV/'+file.replace('.pcap','.csv'),index=False)
except Exception as e:
print(e)
print('# collecting dataframe\n')
r,c = join()
print('\nGOT ',r,'nonzero records out of ',len(os.listdir('TEMP')),'files\n')
print('df shape: ',r,'x',c)
mk_dir()
def selfcal_output(self, overwrite=False):
outputvis = self.visfile + '.selfcal'
if overwrite:
os.system('rm -rf ' + outputvis)
split(vis=self.visfile, outputvis=outputvis, datacolumn='corrected')
return outputvis
def gen_farpos(set_name):
name = "%s_neg_farpos.mat" % set_name
dat = np.array(gen_negative_set.far_pos("indexes/%s_calc.mat" % set_name))
train_file = os.path.join(FOLDERS["train"], name)
val_file = os.path.join(FOLDERS["val"], name)
test_file = os.path.join(FOLDERS["test"], name)
split(dat, train_file, val_file, test_file, .05, .10)
def gen_closepairs(set_name):
name = "%s_neg_closepairs.mat" % set_name
dat = np.array(gen_negative_set.closepairs("indexes/%s_file_index.mat" % set_name))
train_file = os.path.join(FOLDERS["train"], name)
val_file = os.path.join(FOLDERS["val"], name)
test_file = os.path.join(FOLDERS["test"], name)
split(dat, train_file, val_file, test_file, .05, .10)
def gen_pos_gta():
l = np.load("indexes/gta_pairs.npy")
split(l,
"indexes/train/gta_pos.npy",
"indexes/val/gta_pos.npy",
"indexes/test/gta_pos.npy",
.05, .10)
def gen(index_path, name):
with open(index_path, "r") as f:
leng = np.load(f).shape[0]
split(np.expand_dims(np.array(range(leng)), 1),
"indexes/train/"+name+"_pos_prox.mat",
"indexes/val/"+name+"_pos_prox.mat",
"indexes/test/"+name+"_pos_prox.mat",
.05, .10)
def test_split():
spl = split(tokenize_and_fix_macros("5 = 6 \\\\ = 6 + 7"))
assert spl == [[['5'], ['6'], ['6', '+', '7']]]
s = '\int_0^{\infty}\\frac{r^{2n-3}dr}{(1+r^2)^{2n-1}} = \\frac{1}{2}\int_0^{\infty}\\frac{r^{n-2}dr}{(1+r)^{2n-1}} = \\frac{1}{2n-2} {2n-2 \choose n-1}^{-1}'
tok = tokenize_and_fix_macros(s)
spl = split(tok)
assert len(spl) == 1 and len(spl[0]) == 3
def main(root_dir):
for js, name in travel_dir(root_dir):
p = './washed/%s' % name
print p
split(js, p)
command = './run/indexer.out -d ../../cppjieba/dict -o %s -p %s' % (
name, p)
ret = subprocess.call(command, shell=True)
print ret, js, name
return 0
def gen_cnn():
l = cnn.gen_pairs("train")
split(l,
"indexes/train/cnn_pos.mat",
"indexes/val/cnn_pos.mat",
"indexes/test/cnn_pos.mat",
.00, .10)
split(cnn.gen_pairs("val"),
"indexes/train/cnn_pos.mat",
"indexes/val/cnn_pos.mat",
"indexes/test/cnn_pos.mat",
1.00, 0.0)
def build_fragments(df, args):
xc_id = '100113'
xc.convert_mp3_to_wav(xc_id)
in_file = xc.get_wav_file(xc_id)
out_path = get_fragments_path(df, xc_id)
os.makedirs(out_path, exist_ok=True)
print(in_file, 'file://' + out_path)
args['output_dir'] = out_path
split.split(in_file, args)
def readDBandSplit(dbfile):
r = reader.tabSepReader(dbfile)
fulltrain, testDict = split.split(
r.getR(), SEED1)
trainingDict, evalDict = split.split(
fulltrain, SEED2)
output = open("splits.npz", "wb")
cPickle.dump(r, output, -1)
cPickle.dump(trainingDict, output, -1)
cPickle.dump(fulltrain, output, -1)
cPickle.dump(testDict, output, -1)
cPickle.dump(evalDict, output, -1)
output.close()
return r, trainingDict, fulltrain, testDict, evalDict
def main():
result = pyfiglet.figlet_format("FILE SHARDER", font="slant")
print(result)
print("Welcome to the file-split cryptography")
filename = str(input("Enter the filepath : "))
if filename == "":
print("Please enter the name of the file : ")
pwd = str(input("Enter the password: "******"Enter the mode for the file operation (read/create): "))
if mode == "read":
recreate_file(filename, pwd)
elif mode == "create":
split(filename, pwd, 1)
else:
print("Wrong Option. Try again !")
def find_question():
red_img = ImageGrab.grab((22, 43, 37, 56))
red_img.save("red_img.gif")
size = red_img.size
red_count = 0
for x in range(size[0]):
for y in range(size[1]):
pix = red_img.getpixel((x, y))
if pix[0] >= 240 and pix[1] <= 10 and pix[1] <= 10:
red_count += 1
if red_count > 40:
#if True:
im = ImageGrab.grab((57, 172, 324, 188))
im.save("question.gif")
result = split.split(im)
index = 0
length = 80
for item in result:
print index
im = Image.open(item)
im_len = im.size[0] + 25
length += im_len
if imghash.match(item, 'dic') == True:
length -= im_len / 2
break
index += 1
print "\n"
print "length:%d"%length
m.click(length, 180)
def compare_errors(k_vals, input_data_file):
## read in the input data
initial_data = create_data(input_data_file)
## create plots of the data (this should save the images within the current
## directory)
plot_data(initial_data)
## integerize the data labels
integerized_data, label_dict = integerize_labels(initial_data)
## split the data into train and test
train, test = split(integerized_data)
## compute the errors
errors = {}
for k in k_vals:
predicted_labels = knn(train, test, k)
error_rate = calculate_error_rate(predicted_labels, test)
errors[k] = error_rate
## BONUS: weighting
for k in k_vals:
weighted_predicted_labels = weighted_knn(train, test, k)
weighted_error_rate = calculate_error_rate(weighted_predicted_labels,
test)
print("Weighted error value for k = %d was %f" %
(k, weighted_error_rate))
return errors
def choose_best_feature_to_split(matrix):
number_of_features = len(matrix[0]) - 1
base_entropy = calentropy.calentropy(matrix)
best_info_gain = 0.0
best_feature = -1
for i in range(number_of_features):
features = [row[i] for row in matrix]
unique_features = set(features)
inst_entropy = 0.0
for feature in unique_features:
splited_matrix = split.split(matrix, i, feature)
probability = len(splited_matrix) / float(len(matrix))
inst_entropy += probability * calentropy.calentropy(splited_matrix)
info_gain = base_entropy - inst_entropy
if (info_gain > best_info_gain):
best_info_gain = info_gain
best_feature = i
return best_feature
def test_22(self):
"""
@Test the maximum for an integer.
"""
expected = ["cs"]
actual = split("cs", 2147483647)
self.assertListEqual(expected, actual)
def ddmin(circumstances, test):
"""Return a sublist of CIRCUMSTANCES that is a relevant configuration
with respect to TEST."""
n = 2
while len(circumstances) >= 2:
subsets = split(circumstances, n)
some_complement_is_failing = 0
for subset in subsets:
complement = listminus(circumstances, subset)
file = open('complement.xml', 'w')
for c in complement:
file.write(c[1])
file.close()
if test('complement.xml') == FAIL:
circumstances = complement
n = max(n - 1, 2)
some_complement_is_failing = 1
break
if not some_complement_is_failing:
if n == len(circumstances):
break
n = min(n * 2, len(circumstances))
os.remove('complement.xml')
return circumstances
def test_8(self):
"""
@Test how the method handles a special character.
"""
expected = ["$"]
actual = split("$", 1)
self.assertListEqual(expected, actual)
def test_21(self):
"""
@Test how the method handles an emtpy string.
"""
expected = [""]
actual = split("", 3)
self.assertListEqual(expected, actual)
def _split_file(header,
path,
dialect=None,
list_columns=None,
list_separator=None,
encoding='utf-8'):
"""Split a CSV file into columns, one column per file.
:arg str header: The names for each column of the file.
:arg str path: The full path to the file.
:arg Dialect dialect: The CSV dialect to use when parsing.
:arg list list_columns:
:arg str list_separator:
Returns the header, the row length histogram, and the paths of the files
storing each column.
"""
#
# This is here because:
#
# 1. This function runs as a separate process, and receives pickled args
# 2. We cannot pickle a csv.Reader
# 3. split.split expects a csv.Reader
#
assert dialect
assert list_separator
with _open_for_reading(path) as fin:
reader = csv.reader(fin, dialect=dialect)
return split.split(header,
reader,
list_columns=list_columns,
list_separator=list_separator,
path=path)
def test_14(self):
"""
@Test how the method handles a logical operation in the string.
"""
expected = ["!"]
actual = split("!", 1)
self.assertListEqual(expected, actual)
def test_12(self):
"""
@Test how the method handles a math operation in the string.
"""
expected = ["/"]
actual = split("/", 1)
self.assertListEqual(expected, actual)
def ddmin(circumstances, test):
"""Return a sublist of CIRCUMSTANCES that is a relevant configuration
with respect to TEST."""
assert test([]) == UNRESOLVED
assert test(circumstances) == FAIL
n = 2
while len(circumstances) >= 2:
subsets = split(circumstances, n)
assert len(subsets) == n
some_complement_is_failing = 0
for subset in subsets:
complement = listminus(circumstances, subset)
if test(complement) == FAIL:
circumstances = complement
n = max(n - 1, 2)
some_complement_is_failing = 1
break
if not some_complement_is_failing:
if n == len(circumstances):
break
n = min(n * 2, len(circumstances))
return circumstances
def test_1(self):
"""
@Test a simple base case to show the program can handle a simple input
"""
expected = ["c", "s"]
actual = split("cs", 1)
self.assertListEqual(expected, actual)
def cv(n, folders, num_cpus, N):
'''
Cross validation by multiprocessing
input:
n - number of samples
folders - number of folders
num_cpus -
N -
'''
# record auc
ROC_AUC = pd.DataFrame()
# record parameters
input_paras = []
# cross validation prepare
for folder in range(folders):
training, testing = split(n, folder, folders)
trte = training, testing
for m in N:
input_paras.append((folder, trte, m))
# cross validation run on multiprocessors
with mp.Pool(processes=num_cpus) as pool:
results_pool = pool.map(single_run, input_paras)
pool.close()
pool.join()
# get results
for folder, m, roc_auc in results_pool:
ROC_AUC[(folder, m)] = roc_auc
return ROC_AUC
def choose_best_feature_to_split(matrix):
number_of_features = len(matrix[0]) - 1
base_entropy = calentropy.calentropy(matrix)
best_info_gain = 0.0
best_feature = -1
for i in range(number_of_features):
features = [row[i] for row in matrix]
unique_features = set(features)
inst_entropy = 0.0
for feature in unique_features:
splited_matrix = split.split(matrix, i, feature)
probability = len(splited_matrix) / float(len(matrix))
inst_entropy += probability * calentropy.calentropy(splited_matrix)
info_gain = base_entropy - inst_entropy
if(info_gain > best_info_gain):
best_info_gain = info_gain
best_feature = i
return best_feature
def ddmin(circumstances, test):
"""Return a sublist of CIRCUMSTANCES that is a relevant configuration
with respect to TEST."""
assert test([]) == PASS
assert test(circumstances) == FAIL
n = 2
while len(circumstances) >= 2:
subsets = split(circumstances, n)
assert len(subsets) == n
some_complement_is_failing = 0
for subset in subsets:
complement = listminus(circumstances, subset)
if test(complement) == FAIL:
circumstances = complement
n = max(n - 1, 2)
some_complement_is_failing = 1
break
if not some_complement_is_failing:
if n == len(circumstances):
break
n = min(n * 2, len(circumstances))
return circumstances
def test_5(self):
"""
@Test how the method handles a negative as the size. Since a string
cannot have negative size, it should raise an error.
"""
with self.assertRaises(ValueError):
actual = split("cs", -1)
def test_9(self):
"""
@Test how the method handles a special character after a string.
"""
expected = ["a", "b", "c", "$"]
actual = split("abc$", 1)
self.assertListEqual(expected, actual)
def solve(heuristic, file):
start = time.time()
heuristics = {0: "Random", 1: "Jeroslaw", 2: "MOMs"}
print("Solving using {} heuristic".format(heuristics[int(heuristic)]))
which_method = int(heuristic)
# List for data analysis EXCEL
# [#unit_clauses, %of reduction from first simplify, #splits, #backtrackings, #time]
results = [0, 0, 100., 0, 0, 0]
split_count = 0
back_track_count = 0
problem_start_time = time.time()
rules = read_files.read_DIMACS_file(file)
rules, literals_dict, truth_values = read_files.init_database(rules)
results[1] = len(truth_values)
old_clauses_count = len(rules)
done = False
rules_before_split, literals_dict_before_split, truth_values_before_split = {}, {}, {}
split_choice, neg_literal = [], []
while done == False:
back_track = False
# Simplify
rules, literals_dict, truth_values, split_choice, neg_literal, \
rules_before_split, literals_dict_before_split, truth_values_before_split, back_track = \
simplify.simplify(rules, literals_dict, truth_values, split_choice, neg_literal,
rules_before_split, literals_dict_before_split, truth_values_before_split, back_track)
new_clauses_count = len(rules)
if split_count > 2000:
break
if back_track:
back_track_count += 1
if new_clauses_count == 0:
pretty_print.solution(truth_values)
results[3] = split_count
results[4] = back_track_count
results[5] = float("{0:.2f}".format(time.time() -
problem_start_time))
print("Solved in {0:.2f}s".format(time.time() -
problem_start_time))
print("# splits {}".format(split_count))
done = True
with open("{}.out".format(file), 'w') as f:
for truth in truth_values:
f.write("{} 0\n".format(truth))
elif old_clauses_count == new_clauses_count and back_track == False:
split_count += 1
# Split
rules, literals_dict, truth_values, split_choice, neg_literal, \
rules_before_split, literals_dict_before_split, truth_values_before_split = \
split.split(rules, literals_dict, truth_values, split_choice, neg_literal,
rules_before_split, literals_dict_before_split, truth_values_before_split, which_method)
old_clauses_count = new_clauses_count
def test_4(self):
"""
@Test how the method functions when given a string and a size
that is greater than the size of that string
"""
expected = ["cs"]
actual = split("cs", 5)
self.assertListEqual(expected, actual)
def test_20(self):
"""
@Test an escape character. We assume these two characters should be
treated as one.
"""
expected = ["\r"]
actual = split("\r", 5)
self.assertListEqual(expected, actual)
def test_6(self):
"""
@Test how the method handles zero as the size. Since the string could
be composed of an infinite number of emtpy strings, it should raise an
error.
"""
with self.assertRaises(ValueError):
actual = split("cs", 0)
def test_3(self):
"""
@Test how the method functions when given a string and a size
that matches the size of that string
"""
expected = ["cs rocks"]
actual = split("cs rocks", 8)
self.assertListEqual(expected, actual)
def ddmin(data, test,n=1):
"""Return a sublist of CIRCUMSTANCES that is a relevant configuration
with respect to TEST."""
subsets=split(data,n)
for subset in subsets:
if test(subset)==FAIL:
return ddmin(subset,test,2)
return tempfile()
def ddmin(data, test, n=1):
isFailed = False
for i in range(n, MAX_DEGREE):
subsets = split(data, i)
for subset in subsets:
if test(subset) == FAIL:
isFailed = True
return ddmin(subset, test, 2)
return gettempfiledata()
def buildClassifier(subset, name, cs, iterations=100, dataDir='data', det=False, verbose=0):
if (verbose >= 1):
print 'Enter buildClassifier'
if det:
random.seed(1)
evalC=[]
bC=0.0
evbC=0.0
mfccs, mfccMatching, _, _, _ = preprocess.preprocess(subset, dataDir=dataDir, verbose=verbose)
y = buildLabels(name, mfccMatching)
dicL, dicV, dicT = split(mfccMatching, name, verbose=verbose)
# learning data
xL=[mfccs[k] for k in np.concatenate(dicL.values())]
yL=[y[k] for k in np.concatenate(dicL.values())]
# validation data
xV=[mfccs[k] for k in np.concatenate(dicV.values())]
yV=[y[k] for k in np.concatenate(dicV.values())]
# testing data
xT=[mfccs[k] for k in np.concatenate(dicT.values())]
yT=[y[k] for k in np.concatenate(dicT.values())]
for c in cs:
if (verbose >= 1):
print 'Processing C: ', c
print 'Learning...'
w = sgd.sgd(xL, yL, np.zeros(len(xL[0])+1), iterations, 1, sgd.L, 0.01, c)
if (verbose >= 1):
print 'Evaluating...'
ev=sgd.eval(xV, yV, w[:-1], w[-1])
evalC.append(ev)
if (ev > evbC):
evbC=ev
bC=c
if (verbose >= 1):
print 'Building classifier with C:', bC, '...'
xL2=xL+xV
yL2=yL+yV
w = sgd.sgd(xL2, yL2, np.zeros(len(xL[0])+1), iterations, 1, sgd.L, 0.01, bC)
if (verbose >= 1):
print 'Evaluating classifier...'
ev=sgd.eval(xT, yT, w[:-1], w[-1])
def f(wavFile):
x = preprocess.mfcc(wavFile)
tot=len(x)
ok=0.0
for i in xrange(tot):
ok += np.dot(w[:-1], x[i]) + w[-1]
return int(ok/tot > 0)
if (verbose >= 1):
print 'Exit buildClassifier'
return f, ev, evalC
def set_image(self, data_bytes):
data = np.frombuffer(data_bytes, dtype="uint8")
cv2im = cv2.imdecode(data, 1)
(cv2green, cv2red) = split(cv2im)
_, green = cv2.imencode(".jpeg", cv2green)
_, red = cv2.imencode(".jpeg", cv2red)
_, st_img = cv2.imencode(".jpeg", stereo(cv2green, cv2red))
self.original_holder.set_image(data)
self.green_holder.set_image(green)
self.red_holder.set_image(red)
self.stereo_holder.set_image(st_img)
def __init__(self, parameters, target, repetitions, db_name, transform=dont_transform):
print("v. 0.3")
fixed_parameters, lower_limits, upper_limits, dtypes = split(parameters)
SimulationSet.setup(dtypes, target, transform)
InputSet.setup(lower_limits, upper_limits, dtypes, fixed_parameters)
self.repetitions = repetitions
self.remote_saudifirms = RemoteSaudiFirms(task=5007, result=5008, kill=5009)
self.local_saudifirms = RemoteSaudiFirms(task=5567, result=5568, kill=5569)
self._db = dataset.connect("sqlite:///%s.sqlite3" % db_name)
parameters = json.dumps(sorted(fixed_parameters.items()))
hash = sha224(parameters).hexdigest()[:7]
overview = self._db['overview']
overview.insert({'hash': hash, 'json_string': parameters})
self.db = self._db[hash]
self.db_name = db_name
def ddmin(circumstances, test, target):
global LOG
"""Return a sublist of CIRCUMSTANCES that is a relevant configuration
with respect to TEST."""
assert test([]) == PASS
assert test(circumstances) == FAIL
n = 2
while len(circumstances) >= 2:
subsets = split(circumstances, n)
assert len(subsets) == n
some_complement_is_failing = 0
for subset in subsets:
complement = listminus(circumstances, subset)
if test(complement) == FAIL:
circumstances = complement
n = max(n - 1, 2)
some_complement_is_failing = 1
break
else:
if test(complement) == FAIL:
LOG.write("undeterministic event detect")
os.system("mv dd_test_tmp undeterministic_input.dat")
p = subprocess.Popen(["bash", 'mt_trace.sh', target], stdin=subprocess.PIPE)
p.communicate()
exit(0)
if not some_complement_is_failing:
if n == len(circumstances):
break
n = min(n * 2, len(circumstances))
return circumstances
def first_first(name):
parts = split(name)
return " ".join([p for p in parts if p])
#Author:Weilong Guo
#英語テキストを行に区切る
import sys
import re
import sys
sys.path.append(r"/Users/kubotanaoyuki/Desktop/NPL/practise/NLP/test3")
from split import split
f = open("medline_sent.txt","r")
print("input your charactor.")
print(" please input \"\w\"":)
s = raw_input("raw_input:")
words = []
for line in f:?
sent = split(line,s)
words += sent
for w in words:
if w.endswith(","):
w = w[:-2]
elif w.endswith("."):
w =w[:-2]+"\n"
print w
def split_and_cal(rootvis, rootprefix, spwn, fluxcal, phasecal, target,
anttable=[], fieldpre=[], fresh_clean=False):
print "Beginning calibration and mapping of spw ", spwn
prefix = rootprefix+"_spw"+spwn
# hopefully this decreases read/write times...
vis = outvis = prefix+".ms"
split(vis=rootvis, outputvis=outvis, spw=spwn, datacolumn='all')
listobs(outvis)
plotants(vis=vis,figfile=prefix+'plotants_'+vis+".png")
#plotms(vis=vis, xaxis='', yaxis='', averagedata=False, transform=False, extendflag=False,
# plotfile='FirstPlot_AmpVsTime.png',selectdata=True,field='')
plotms(vis=vis, spw='0', averagedata=True, avgchannel='64', avgtime='5', xaxis='uvdist', yaxis='amp', field=phasecal, plotfile=prefix+"phasecal_%s_AmpVsUVdist_spw%s.png" % (phasecal,spwn),overwrite=True)
plotms(vis=vis, spw='0', averagedata=True, avgchannel='64', avgtime='5', xaxis='uvdist', yaxis='amp', field=fluxcal, plotfile=prefix+"fluxcal_%s_AmpVsUVdist_spw%s.png" % (fluxcal,spwn),overwrite=True)
plotms(vis=vis, spw='0', averagedata=True, avgchannel='64', avgtime='5', xaxis='uvdist', yaxis='amp', field=target, plotfile=prefix+"target_%s_AmpVsUVdist_spw%s.png" % (target,spwn),overwrite=True)
plotms(vis=vis, field='',correlation='RR,LL',timerange='',antenna='ea01',spw='0',
xaxis='time',yaxis='antenna2',coloraxis='field',plotfile=prefix+'antenna2vsantenna1vstime_spw%s.png' % spwn,overwrite=True)
#gencal(vis=vis,caltable=prefix+".antpos",caltype="antpos")
# this apparently doubles the data size... clearcal(vis=vis,field='',spw='')
#setjy(vis=vis, listmodels=T)
gaincal(vis=vis, caltable=prefix+'.G0all',
field='0,1', refant='ea21', spw='0:32~96',
gaintype='G',calmode='p', solint='int',
minsnr=5, gaintable=anttable)
#didn't work
plotcal(caltable=prefix+'.G0all',xaxis='time',yaxis='phase',
spw='0',
poln='R',plotrange=[-1,-1,-180,180],
figfile=prefix+'.G0all.png')
gaincal(vis=vis, caltable=prefix+'.G0',
field=fluxcal, refant='ea21', spw='0:20~100', calmode='p', solint='int',
minsnr=5, gaintable=anttable)
plotcal(caltable=prefix+'.G0',xaxis='time',yaxis='phase',
spw='0',
poln='R',plotrange=[-1,-1,-180,180],
figfile=prefix+'.G0.png')
gaincal(vis=vis,caltable=prefix+'.K0',
field=fluxcal,refant='ea21',spw='0:5~123',gaintype='K',
solint='inf',combine='scan',minsnr=5,
gaintable=anttable+[
prefix+'.G0'])
plotcal(caltable=prefix+'.K0',xaxis='antenna',yaxis='delay',
spw='0',
figfile=prefix+'.K0_delayvsant_spw'+spwn+'.png')
bandpass(vis=vis,caltable=prefix+'.B0',
field=fluxcal,spw='0',refant='ea21',solnorm=True,combine='scan',
solint='inf',bandtype='B',
gaintable=anttable+[
prefix+'.G0',
prefix+'.K0'])
# In CASA
plotcal(caltable= prefix+'.B0',poln='R',
spw='0',
xaxis='chan',yaxis='amp',field=fluxcal,subplot=221,
figfile=prefix+'plotcal_fluxcal-B0-R-amp_spw'+spwn+'.png')
#
plotcal(caltable= prefix+'.B0',poln='L',
spw='0',
xaxis='chan',yaxis='amp',field=fluxcal,subplot=221,
figfile=prefix+'plotcal_fluxcal-B0-L-amp_spw'+spwn+'.png')
#
plotcal(caltable= prefix+'.B0',poln='R',
spw='0',
xaxis='chan',yaxis='phase',field=fluxcal,subplot=221,
plotrange=[-1,-1,-180,180],
figfile=prefix+'plotcal_fluxcal-B0-R-phase_spw'+spwn+'.png')
#
plotcal(caltable= prefix+'.B0',poln='L',
spw='0',
xaxis='chan',yaxis='phase',field=fluxcal,subplot=221,
plotrange=[-1,-1,-180,180],
figfile=prefix+'plotcal_fluxcal-B0-L-phase_spw'+spwn+'.png')
gaincal(vis=vis,caltable=prefix+'.G1',
field=fluxcal,spw='0:5~123',
solint='inf',refant='ea21',gaintype='G',calmode='ap',solnorm=F,
gaintable=anttable+[
prefix+'.K0',
prefix+'.B0'])
gaincal(vis=vis,caltable=prefix+'.G1',
field=phasecal,
spw='0:5~123',solint='inf',refant='ea21',gaintype='G',calmode='ap',
gaintable=anttable+[
prefix+'.K0',
prefix+'.B0'],
append=True)
plotcal(caltable=prefix+'.G1',xaxis='time',yaxis='phase',
spw='0',
poln='R',plotrange=[-1,-1,-180,180],figfile=prefix+'plotcal_fluxcal-G1-phase-R_spw'+spwn+'.png')
plotcal(caltable=prefix+'.G1',xaxis='time',yaxis='phase',
spw='0',
poln='L',plotrange=[-1,-1,-180,180],figfile=prefix+'plotcal_fluxcal-G1-phase-L_spw'+spwn+'.png')
plotcal(caltable=prefix+'.G1',xaxis='time',yaxis='amp',
spw='0',
poln='R',figfile=prefix+'plotcal_fluxcal-G1-amp-R_spw'+spwn+'.png')
plotcal(caltable=prefix+'.G1',xaxis='time',yaxis='amp',
spw='0',
poln='L',figfile=prefix+'plotcal_fluxcal-G1-amp-L_spw'+spwn+'.png')
myscale = fluxscale(vis=vis,
caltable=prefix+'.G1',
fluxtable=prefix+'.fluxscale1',
reference=[fluxcal],
transfer=[phasecal])
applycal(vis=vis,
field=fluxcal,
spw='0',
gaintable=anttable+[
prefix+'.fluxscale1',
prefix+'.K0',
prefix+'.B0',
],
gainfield=fieldpre+[fluxcal,'',''],
interp=fieldpre+['nearest','',''],
calwt=F)
applycal(vis=vis,
field=phasecal,
spw='0',
gaintable=anttable+[
prefix+'.fluxscale1',
prefix+'.K0',
prefix+'.B0',
],
gainfield=fieldpre+[phasecal,'',''],
interp=fieldpre+['nearest','',''],
calwt=F)
applycal(vis=vis,
field=target,
spw='0',
gaintable=anttable+[
prefix+'.fluxscale1',
prefix+'.K0',
prefix+'.B0',
],
gainfield=fieldpre+[phasecal,'',''],
interp=fieldpre+['linear','',''],
calwt=F)
plotms(vis=prefix+'.ms',field=fluxcal,correlation='',
spw='0',
antenna='',avgtime='60s',
xaxis='channel',yaxis='amp',ydatacolumn='corrected',
plotfile=prefix+'_fluxcal-corrected-amp_spw'+spwn+'.png',overwrite=True)
#
plotms(vis=prefix+'.ms',field=fluxcal,correlation='',
spw='0',
antenna='',avgtime='60s',
xaxis='channel',yaxis='phase',ydatacolumn='corrected',
plotrange=[-1,-1,-180,180],coloraxis='corr',
plotfile=prefix+'_fluxcal-corrected-phase_spw'+spwn+'.png',overwrite=True)
#
plotms(vis=prefix+'.ms',field=phasecal,correlation='RR,LL',
spw='0',
timerange='',antenna='',avgtime='60s',
xaxis='channel',yaxis='amp',ydatacolumn='corrected',
plotfile=prefix+'_phasecal-corrected-amp_spw'+spwn+'.png',overwrite=True)
#
plotms(vis=prefix+'.ms',field=phasecal,correlation='RR,LL',
spw='0',
timerange='',antenna='',avgtime='60s',
xaxis='channel',yaxis='phase',ydatacolumn='corrected',
plotrange=[-1,-1,-180,180],coloraxis='corr',
plotfile=prefix+'_phasecal-corrected-phase_spw'+spwn+'.png',overwrite=True)
plotms(vis=prefix+'.ms',field=phasecal,correlation='RR,LL',
spw='0',
timerange='',antenna='',avgtime='60s',
xaxis='phase',xdatacolumn='corrected',yaxis='amp',ydatacolumn='corrected',
plotrange=[-180,180,0,3],coloraxis='corr',
plotfile=prefix+'_phasecal-corrected-ampvsphase_spw'+spwn+'.png',overwrite=True)
plotms(vis=vis,xaxis='uvwave',yaxis='amp',
spw='0',
field=fluxcal,avgtime='30s',correlation='RR',
plotfile=prefix+'_fluxcal-mosaic0-uvwave_spw'+spwn+'.png',overwrite=True)
plotms(vis=vis,xaxis='uvwave',yaxis='amp',
spw='0',
field=target,avgtime='30s',correlation='RR',
plotfile=prefix+'_target-mosaic0-uvwave_spw'+spwn+'.png',overwrite=True)
for antenna in xrange(28):
plotms(vis=vis, xaxis='time', yaxis='amp', spw='0',
field='', avgchannel='64', coloraxis='corr',
avgtime='5s',
antenna='%i' % antenna,
plotfile=prefix+"_antenna%02i_ampVStime.png" % antenna,
overwrite=True)
imagename = prefix+"_mfs"
if fresh_clean:
shutil.rmtree(imagename+".model")
shutil.rmtree(imagename+".image")
shutil.rmtree(imagename+".psf")
shutil.rmtree(imagename+".flux")
shutil.rmtree(imagename+".residual")
clean(vis=outvis,
imagename=imagename,
field=target,spw='',
mode='mfs', # use channel to get cubes
nterms=1, # no linear polynomial
niter=5000,
gain=0.1, threshold='1.0mJy',
psfmode='clark',
multiscale=[0],
interactive=False,
imsize=[2560,2560],
cell=['0.1arcsec','0.1arcsec'],
stokes='I',
weighting='uniform',
allowchunk=True,
usescratch=True)
exportfits(imagename=imagename+".image", fitsimage=imagename+".fits", overwrite=True)
print "Finished spw ",spwn
def initial_initial(name):
pre, fp, lp, suff = split(name)
if fp:
fp = fp[0]
return " ".join([p for p in [pre, fp, lp, suff] if p])
import split splitFile = '../records/test2.split.json' #scoreFile = '../score/test2.mid' scoreFile = '../score/mahler.mid' split.split(splitFile, scoreFile)
def split(args):
import split
split.split(args)
When you use this program, you need to point out
1. mainDirectory: The directory of your Alchemy application, where you put your .mln and .db files into
2. AlchemyBinDirectory: The bin directory of Alchemy
3. divideNumber: The number you need to cross-validation
"""
"""
This program run by:
1. split the HasWord(word, page) and Topic(class, page) in the file text-class-train.db
2. turn the HasWord.db and Links.db into dict for efficient searching
3. split the database into training set and testing set
4. run Alchemy weight-learning and infer for each cross-validation(multiprocessing)
"""
whoami = subprocess.check_output("whoami", shell=True).split('\n')[0]
if whoami == "panjinbo":
mainDirectory = "/home/panjinbo/software/alchemy2-master/tutorial/Section 5 - Text Classification/"
AlchemyBinDirectory = "/home/panjinbo/software/alchemy2-master/bin/"
else:
mainDirectory = "/home/lindayong/alchemy2-master/tutorial/Section 5 - Text Classification/"
AlchemyBinDirectory = "/home/lindayong/alchemy2-master/bin/"
divideNumber = 5
os.chdir(mainDirectory)
split()
database = turnToDict()
matchPages(divideNumber, database)
run(AlchemyBinDirectory, divideNumber)
analyze(divideNumber)
#!/usr/bin/python
# encoding: utf-8
import sys
import unigram
import split
if __name__ == "__main__":
if len(sys.argv) < 3:
print("usage: %s train_file split_target")
sys.exit(1)
model1 = unigram.load_model(sys.argv[1])
print(model1)
infile = open(sys.argv[2], 'r')
for line in infile:
line = line.strip()
ustr = unicode(line, "utf-8")
print("splitting: %s" % ustr)
word = split.split(model1, ustr)
def testQP(nbGs, nC=2, nS=100, nF=1000, N=10, m=[-50.0, 0.0], M=[0.0, 50.0], det=False, verbose=0):
if det:
t=1
else:
t=time.clock()
evalG=[]
bG=0
evbG=0.0
name='1'
for nbG in nbGs:
if (verbose >= 1):
print 'Processing with '+str(nbG)+' gaussians...'
random.seed(t)
mfccs, _, mu, pi, gmmMatching = buildData(nC, nS, nF, N, m, M, nbG)
if (verbose >= 2):
print gmmMatching
y = bl.buildLabels2(name, gmmMatching)
dicL, dicV, dicT = split.split(gmmMatching, name, verbose=verbose)
# learning data
dicLv=np.concatenate(dicL.values())
xL=[j for (j,_,_) in dicLv]
yL=[y[j] for (j,_,_) in dicLv]
# validation data
dicVv=np.concatenate(dicV.values())
xV=[j for (j,_,_) in dicVv]
yV=[y[j] for (j,_,_) in dicVv]
# testing data
dicTv=np.concatenate(dicT.values())
xT=[j for (k,_,_) in dicTv]
yT=[y[j] for (j,_,_) in dicTv]
mfccChosen = np.concatenate([range(u,v) for (_,u,v) in dicLv])
if (verbose >= 1):
print 'xL:', np.shape(xL)
print 'dicLv:', np.shape(dicLv)
print 'mfccs:', np.shape(mfccs)
print mfccChosen
for j in mfccChosen:
if j not in xrange(len(mfccs)):
print "voilà le méchant: ",j
g = pp.gmm([mfccs[j] for j in np.concatenate([range(u,v) for (_,u,v) in dicLv])], nbG)
mu0=g.means
sig0=g.covars
def k(i,j):
res = fisher.K(xL,i,j,mu[i],mu[j],sig0,pi[i],pi[j],mu0)
return res
if (verbose >= 1):
print 'Learning...'
print 'Evaluating...'
# w, b, ev = qp.qp(xL, yL, xV, yV, k, 1.0)
# ev=ev[0][0]
# evalG.append(ev)
# if (ev > evbG):
# evbG=ev
# bG=nbG
if (verbose >= 1):
print 'Building classifier with '+str(bG)+' gaussians...'
xL2=xL+xV
yL2=yL+yV
dicL2v=np.concatenate([dicLv,dicVv])
g2 = pp.gmm([mfccs[k] for k in np.concatenate([range(u,v) for (_,u,v) in dicL2v])], bG)
mu02=g2.means
sig02=g2.covars
if (verbose >= 1):
print len(dicL2v), np.shape(mu), np.shape(sig02)
def k2(i,j):
res = fisher.K(dicL2v,i,j,mu[i],mu[j],sig02,pi[i],pi[j],mu02)
return res
if (verbose >= 1):
print 'Learning...'
print 'Evaluating...'
____________MAIN___________
"""
Jer = {
"+":1,
"-":1,
"*":2,
"/":2,
"^":3,
")":0,
"(":6
}
print ("Simbolos aceptados:")
simb = "\t Numeros Enteros \n\t Letras Mayusculas \n\t Letras Minusculas \n\t Operadres : + - * / ^ \n\t Separadores : ( )\n"
print (simb)
exp = input ("Ingresa la exp \n")
arr= split(exp)
if arr==[]:
exit()
oper, num , var = typ(arr)
#print ("arr" , arr)
print ("opers", oper)
print ("number", num)
#print ("vars", var)
if len(var) > 0:
print("Se encontaron varibales ")
var = takenvar(var)
print("var", var)
pos = convert(arr, Jer)
def last_first(name):
pre, fp, lp, suf = split(name)
if lp:
lp += ", "
return lp + " ".join([p for p in [pre, fp, suf] if p])
Skip(lh.leaves[168])
for leaf in lh.leaves[245 : 281 + 1]:
Skip(leaf)
### dynamic line spanner management ###
for override in lh.spanners.get(grob="DynamicLineSpanner"):
override.die()
### split rh chords ###
### NOTE: CHANGES LEAF COUNT AND MUST HAPPEN LAST
for leaf in rh.leaves[730 : 771 + 1] + rh.leaves[777 : 779 + 1] + rh.leaves[782 : 785 + 1]:
split(leaf, lh._parent)
for leaf in rh.leaves[809 : 811 + 1]:
split(leaf, lh._parent)
### number measures ###
for staff in staves:
for voice in staff:
for i, measure in enumerate(voice):
measure.comments.before.append("measure %s" % (i + 1))
### remove original leaf numbering ###
unnumber(staves)
for leaf in instances(staves, "_Leaf"):
def last_only(name):
return split(name)[2]
print "xmlproc version %s" % xmlproc.version
for sysid in sysids:
print
print "Parsing '%s'" % sysid
p.set_data_after_wf_error(0)
try:
p.parse_resource(sysid)
except UnboundLocalError:
print "FAIL"
open_file=open(sysid,'r')
for line in open_file:
#print(line.strip())
a=split(line.strip(),4)
print a
open_file.close()
except:
print "UNRESOLVED"
else:
print "PASS!"
print "Parse complete, %d error(s)" % err.errors,
if warnings:
print "and %d warning(s)" % err.warnings
else:
print
