def main():
# query = " ".join(sys.argv[1:])
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="action",
help="use \"[command] -h\" to see help message for individual command")
parser_clone = subparsers.add_parser('clone')
parser_clone.add_argument('resource_type', type=str,
choices=['caseversion', 'case', 'suite'], help="Resource type")
parser_clone.add_argument('id', type=int, help="Resource ID")
parser_diff = subparsers.add_parser('diff')
parser_diff.add_argument('filename', type=str, help="File to be diffed")
parser_push = subparsers.add_parser('push')
parser_push.add_argument('-f', '--force', action="store_true",
help="Force overwrite (BE CAREFUL!)", required=True)
parser_push.add_argument('filename', type=str, help="File to be pushed")
parser_push.add_argument('-u', '--username',
help="MozTrap username", required=True)
parser_push.add_argument('-k', '--api_key',
help="MozTrap api key", required=True)
args = parser.parse_args()
# print args
if args.action == "clone":
mtapi.clone(args.resource_type, args.id)
elif args.action == "diff":
diff.diff(args)
elif args.action == "push":
#if not args.force:
# raise Exception("Push will force override everything on the server."
# + " Use \"push -f\" to acknowledge the risk")
credental = {'username': args.username, 'api_key': args.api_key}
mtapi.push(args.filename, credental)
def chose_target(self, chrl, tracer):
for i in range(len(chrl)):
ch = chrl[i]["data"]["header"]["firstLine"]
for j in range(len(chrl[i]["data"]["header"]["headers"])):
ch_p = chrl[i]["data"]["header"]["headers"][j]
ch_pp = ch_p["name"] + " " + ch_p["value"]
ch = ch + ch_pp + "\r\n"
ch_time = chrl[i]["infos"]["time"][11:-10]
# hex_ch = utf8_decoder(ch)
temp = {"data": ch, "time": ch_time}
self.target_ch.append(temp)
for g in range(len(tracer)):
tr_type = TARGETAPI_PROFILESLIST.index(tracer[g]["api_infos"]["trace_api"])
'''
api_target = tracer[g]["data"][SENDAPI_PROFILES[TARGETAPI_PROFILESLIST[tr_type]]["send_data"]]
index = api_target.find("000000000000000000000000000000000000000000000000")
if not index == -1:
api_target = api_target[:index - len(api_target)]
'''
api_target = tracer[g][SENDAPI_PROFILES[TARGETAPI_PROFILESLIST[tr_type]]["data_string"]]
tr_time = tracer[g]["api_infos"][SENDAPI_PROFILES[TARGETAPI_PROFILESLIST[tr_type]]["time"]][:-4]
temp1 = {"data": api_target, "time": tr_time}
self.target_tracer.append(temp1)
diff.diff(self.target_ch, self.target_tracer, self.diff_cmp, diff.default_print_result)
def compare_run(request):
if request.method == 'GET':
return render(request, 'show/compare_run.html')
if request.method == 'POST':
compare_hostname = ""
if request.POST.has_key('hostname'):
compare_hostname = request.POST['hostname']
matched_items = []
for item in os.listdir('/etc/ansible/showsite/show/show_outputs'):
match = compare_hostname + ".txt"
if re.search(match, item) != None:
matched_items.append(item)
return JsonResponse({'matched_items': matched_items})
else:
line_total = request.POST['lines']
lines = re.split('\n', line_total)
lines = lines[0: (len(lines) - 1)]
path = '/etc/ansible/showsite/show/show_outputs/'
path_comp = '/etc/ansible/showsite/show/compare_outputs/'
outputfilename = ""
items = re.split(' ', str(datetime.datetime.now()))
for item in re.split(' ', str(datetime.datetime.now())):
outputfilename = outputfilename + item + "_"
outputfilename = "compare" + outputfilename + host_user
diff.diff(path+lines[0], path+lines[1], path_comp+outputfilename)
return JsonResponse({'data': outputfilename})
def main():
f = lambda x: x**3
xF = range(-20, 20)
yF = [f(x) for x in xF]
yf = [diff(f, x) for x in xF]
plot(xF, yF)
plot(xF, yf)
show()
print diff(f, 20)
def test_deleted_text(self):
original_list = ["line 1", "line 2"]
head_modified_list = ["line 2"]
tail_modified_list = ["line 1"]
expected_result = [DiffResult(0, 1, 0, "line 1")]
actual_result = diff(original_list, head_modified_list)
assert expected_result == actual_result
expected_result = [DiffResult(0, 2, 1, "line 2")]
actual_result = diff(original_list, tail_modified_list)
assert expected_result == actual_result
def main():
parser = build_arg_parser()
args = parser.parse_args()
if (args.sub_command == 'push'):
if (len(args.files) == 0):
logger.die('Must include at least one file')
else:
for f in args.files:
upload.upload(f, args.MediaFire_Path)
elif (args.sub_command == 'pull'):
if (len(args.files) == 0):
logger.die('Must include at least one file')
else:
for f in args.files:
download.download(f)
elif (args.sub_command == 'del'):
if (len(args.files) == 0):
logger.die('Must include at least one file')
else:
for f in args.files:
delete.delete(f)
elif (args.sub_command == 'init'):
if (user.is_user_signed_in()):
logger.end('User is already initialized')
else:
user.get_auth()
elif (args.sub_command == 'list'):
if (len(args.files) == 0):
lister.list_files('')
else:
for f in args.files:
lister.list_files(f)
elif (args.sub_command == 'diff'):
if (len(args.files) == 0):
logger.die('Must include at least one file')
else:
for f in args.files:
diff.diff(f, args.MediaFire_Path)
elif (args.sub_command == 'out'):
user.log_out()
elif (args.sub_command == 'change'):
user.change_user()
elif (args.sub_command == 'share'):
if (len(args.files) == 0):
logger.die('Must include at least on file')
else:
for f in args.files:
share.share(f)
def main():
parser = build_arg_parser()
args = parser.parse_args()
if (args.sub_command == 'push'):
if (len(args.files) == 0):
logger.die('Must include at least one file')
else:
for f in args.files:
upload.upload(f, args.MediaFire_Path)
elif (args.sub_command == 'pull'):
if (len(args.files) == 0):
logger.die('Must include at least one file')
else:
for f in args.files:
download.download(f)
elif (args.sub_command == 'del'):
if (len(args.files) == 0):
logger.die('Must include at least one file')
else:
for f in args.files:
delete.delete(f)
elif (args.sub_command == 'init'):
if (user.is_user_signed_in()):
logger.end('User is already initialized')
else:
user.get_auth()
elif (args.sub_command == 'list'):
if (len(args.files) == 0):
lister.list_files('')
else:
for f in args.files:
lister.list_files(f)
elif (args.sub_command == 'diff'):
if (len(args.files) == 0):
logger.die('Must include at least one file')
else:
for f in args.files:
diff.diff(f, args.MediaFire_Path)
elif (args.sub_command == 'out'):
user.log_out()
elif (args.sub_command == 'change'):
user.change_user()
elif (args.sub_command == 'share'):
if (len(args.files) == 0):
logger.die('Must include at least on file')
else:
for f in args.files:
share.share(f)
def main():
parser = optparse.OptionParser("%prog - I diff repo manifests")
parser.add_option("--filter", dest="filters", action="append")
parser.add_option("--diff", "-d", dest="diff", action="store_false", default=True)
parser.add_option("--format", dest="out_format", default="report")
parser.add_option("--output", dest="output", default=None)
parser.add_option("--root", dest="root", default=os.getcwd())
options, args = parser.parse_args()
if not options.output:
output = sys.stdout
else:
output = options.output
if os.path.exists(output):
print >> sys.stderr, "ERROR: Output file already exists"
exit(1)
if len(args) == 0:
print "Choose a command: diff, cleanup, filter"
exit(1)
elif len(args) > 1:
cmd_args = args[1:]
else:
cmd_args = None
cmd = args[0]
if cmd == 'diff':
if len(cmd_args) != 2:
print >> sys.stderr, "ERROR: must specify exactly two arguments (left and right)"
exit(1)
diff(cmd_args[0], cmd_args[1], output=output, output_format=options.out_format, filters=options.filters)
elif cmd == 'freeze':
freeze(cmd_args[0], output, options.root,
gaia_branch='v1-train',
gecko_branch='gecko-18',
moz_remotes=['b2g'],
moz_branch='v1-train')
elif cmd == 'cleanup':
if len(cmd_args) != 1:
print >> sys.stderr, "ERROR: you can only filter one file at a time"
exit(1)
cleanup(cmd_args[0], output, options.filters)
elif cmd == 'filter':
if len(cmd_args) != 1:
print >> sys.stderr, "ERROR: you can only filter one file at a time"
exit(1)
if options.filters == None:
print >> sys.stderr, "ERROR: you must specify filters for the filter command"
exit(1)
filter(cmd_args[0], output, options.filters)
def xmldiff(options):
n = 0
for file1, file2 in diff.gather_files('*.xml', options.source,
options.destination):
n += diff.diff(file1, file2, options)
return n
def diff(self, *w, **kw):
''' Jagare's diff wrapper '''
try:
kws = {}
ignore_space = kw.get('ignore_space', None)
if ignore_space:
flags = kw.get('flags', 0)
flags |= GIT_DIFF_IGNORE_WHITESPACE
kws.update({'flags': flags})
from_ref = kw.get('from_ref', None)
if from_ref:
kws.update({'from_ref': from_ref})
context_lines = kw.get('context_lines', None)
if context_lines:
kws.update({'context_lines': context_lines})
path = kw.get('path', None)
paths = kw.get('paths', None)
if path:
kws.update({'paths': [path]})
if paths:
kws.update({'paths': paths})
# call diff
d = diff(self.repository, *w, **kws)
rename_detection = kw.get('rename_detection', None)
if rename_detection:
d['diff'].find_similar()
#d.find_similar()
# return formated diff dict
return format_diff(d)
except JagareError:
return []
def main(first_file_path: str, second_file_path: str):
with open(first_file_path, "r") as fa:
with open(second_file_path, "r") as fb:
first_file_lines = [line.strip() for line in fa.readlines()]
second_file_lines = [line.strip() for line in fb.readlines()]
diff_list = diff(first_file_lines, second_file_lines)
diff_item_list = []
for flag, old_line_number, new_line_number, text in diff_list:
if len(diff_item_list) == 0 or (
abs(old_line_number - diff_item_list[-1][1]) <= 1
and abs(new_line_number - diff_item_list[-1][2]) <= 1
):
diff_item_list.append(
(flag, old_line_number, new_line_number, text)
)
continue
print_diff(diff_item_list)
diff_item_list.clear()
diff_item_list.append((flag, old_line_number, new_line_number, text))
if len(diff_item_list) > 0:
print_diff(diff_item_list)
def newton(f, t, x):
epsilon = .001
for i in range(t):
prev_x = x
x = x - (f(x) / diff(f, x))
if abs(x - prev_x) < epsilon:
return x, i
return x, t
def test_same_text(self):
original_list = ["aaaaa"]
modified_list = ["aaaaa"]
expected_result = []
actual_result = diff(original_list, modified_list)
assert expected_result == actual_result
def test(self):
actual = str(diff([], []))
expected = str([])
self.assertEqual(actual, expected)
actual = str(diff(["foo", "bar"], []))
expected = "[(/ [foo, bar], [])]"
self.assertEqual(actual, expected)
actual = str(diff([], ["foo", "bar"]))
expected = "[(/ [], [foo, bar])]"
self.assertEqual(actual, expected)
actual = str(diff(["foo", "bar"], ["foo", "bar"]))
expected = "[(= [foo, bar], [foo, bar])]"
self.assertEqual(actual, expected)
actual = str(diff(["foo", "bar"], ["foo", "baz"]))
expected = "[(= [foo], [foo]), (/ [bar], [baz])]"
self.assertEqual(actual, expected)
actual = str(diff(["foo", "bar"], ["foo", "baz", "bar"]))
expected = "[(= [foo], [foo]), (/ [], [baz]), (= [bar], [bar])]"
self.assertEqual(actual, expected)
actual = str(diff(["foo", "baz", "bar"], ["foo", "bar"]))
expected = "[(= [foo], [foo]), (/ [baz], []), (= [bar], [bar])]"
self.assertEqual(actual, expected)
def showDifference(self, zope, identifier, objectid):
""" return a nice formatted difference string """
code_source = self.getSourcecode(identifier)
object = getattr(zope, objectid)
assert hasattr(object, 'absolute_url'), "%r not a ZODB object" % object
code_object = object.document_src()
#difference = diff(html_quote(code_source), html_quote(code_object))
difference = diff(code_object, code_source)
return difference # this is a HTML <table>
def newton_h(expr, guess, precision, niter):
assert isinstance(expr, Expr)
assert type(guess) == float
assert type(precision) == float
assert precision >= 0
assert type(niter) == int
assert niter >= 0
if niter == 0:
if abs(eval_expr(expr, guess)) < precision:
return guess
# no valid root found
return None
if abs(eval_expr(expr, guess)) < precision:
return guess
new_guess = guess - eval_expr(expr, guess) / (
eval_expr(diff(expr), guess) if eval_expr(diff(expr), guess) != 0 else 10E-15)
return newton_h(expr, new_guess, precision, niter - 1)
def test_single_line_diff(self):
original_list = ["aaaaa"]
modified_list = ["bbbbb"]
expected_result = [
DiffResult(0, 1, 0, "aaaaa"),
DiffResult(1, 1, 1, "bbbbb"),
]
actual_result = diff(original_list, modified_list)
assert expected_result == actual_result
def runResample(t):
M = t.shape[0]
x = torch.linspace(0, 1, N)
out = torch.zeros((N,))
dt = diff(t)
cub = [cubic(t[i], t[i + 1], dt[i], dt[i + 1]) \
for i in range(M - 1)]
for i, xi in enumerate(x):
y = xi * (M - 1)
j = min(math.floor(y), M - 2)
dy = y - j
out[i] = sum(cub[j][k] * dy ** k for k in range(4))
return out
def predict_ar(day, data):
firsr_order = 12
second_order = 1
temp1 = data[0:firsr_order]
data = diff.diff(firsr_order, data)
data = diff.diff(second_order, data[firsr_order:])
meandata = arma_math.mean_value(data)
data = arma.decent(meandata, data)
phi = arma.AR(data[second_order:])
for i in range(0, day):
j = -1
tem_sum = 0
for value in phi:
tem_sum += value[0] * data[j]
j -= 1
data.append(tem_sum)
data = arma.cent(meandata, data)
data = diff.rediff(second_order, data)
data = diff.rediff(firsr_order, temp1 + data)
return round(data[-1], 4)
def f_sp(state, output):
rho, ru, p = state
u = ru / rho
or2 = over_r2
diffu = diff(u)
mu_damped = get_damped_viscosity(state, diffu)
output[0] = -or2 * diff(r2 * ru)
output[1] = -or2 * diff(r2 * ru**2 / rho) - diff(p) + 2*diff(mu_damped * diffu)
output[2] = -or2 * diff(r2 * p*u) - (gamma - 1) * p*or2*diff(r2*u) + 2*(gamma - 1)*mu_damped*diffu**2
def f_cy(state, output):
(rho, ru, p) = state
u = ru / rho
diffu = diff(u)
mu_damped = get_damped_viscosity(state, diffu)
output[0] = -over_r*diff(r*ru)
output[1] = -over_r*diff(r * ru**2 / rho) - diff(p) + 2*diff(mu_damped * diffu)
output[2] = -over_r*diff(r * p*u) - (gamma - 1) * p*over_r*diff(r*u) + 2*(gamma - 1)*mu_damped*diffu**2
def predict_ar(last_day, day1, day2, data):
#data = update(data)
data = arma_math.fil(data)
data = [a**(1. / 2) for a in data]
firsr_order = 1
second_order = 7
temp1 = data[0:firsr_order]
data = diff.diff(firsr_order, data)
data = diff.diff(second_order, data[firsr_order:])
meandata = arma_math.mean_value(data)
data = arma.decent(meandata, data)
var = arma_math.var(data)
try:
data = [a / var for a in data]
except ZeroDivisionError:
var = 1
phi = arma.AR(data[second_order:])
#phi,theta = arma.ARMA(data[second_order:])
last_day = datetime.date(
int(last_day[-20:-16]),
int(last_day[-15:-13]),
int(last_day[-12:-10]),
)
d1 = datetime.date(int(day1[:4]), int(day1[5:7]), int(day1[8:10]))
d2 = datetime.date(int(day2[:4]), int(day2[5:7]), int(day2[8:10]))
data = ar_predict(data, phi, (d2 - d1).days)
#data = arma_predict(data,phi,theta,(d2-d1).days)
data = [a * var for a in data]
data = arma.cent(meandata, data)
data = diff.rediff(second_order, data)
data = diff.rediff(firsr_order, temp1 + data)
for i in range(len(data)):
if data[i] < 0:
data[i] = 0
data = [a**2 for a in data]
absList = map(round, data[-(d2 - d1).days:])
return round(sum(data[-(d2 - d1).days:]))
def test_with_dummy_file(self):
with open("tests/data_a.txt", "r") as fa:
with open("tests/data_b.txt", "r") as fb:
file_a_lines = [line.strip() for line in fa.readlines()]
file_b_lines = [line.strip() for line in fb.readlines()]
expected_result = [
DiffResult(0, 1, 0, "Lorem ipsum dolor sit amet"),
DiffResult(0, 3, 1, "Integer at tortor"),
DiffResult(0, 4, 1, "auctor, eleifend magna et,"),
DiffResult(1, 4, 2, "Integer tortor"),
DiffResult(1, 5, 4, ""),
DiffResult(1, 5, 5, "Nullam dapibus libero"),
]
actual_result = diff(file_a_lines, file_b_lines)
assert expected_result == actual_result
def test_single_case(self, execution, time_limit, input_path, answer_path):
# Args:
# execution (str)
# time_limit (int): time limit (s)
# input_path (pathlib.PosixPath): path to input file
# answer_path (pathlib.PosixPath): path to input file
#
# Returns (float):
# maximum time duration (s)
output_path = Path('output')
start_time = time.time()
exec_code = self.command(
['timeout', '-s', '9', str(time_limit), execution],
stdin=self.get_path(input_path).open(),
stdout=self.get_path(output_path).open('w'))
time_duration = time.time() - start_time
# Check judge status
if exec_code in [137, 35072]:
self.show_result('Time Limit Exceeded',
input_path, time_duration, 'yellow')
self.show_file(input_path)
self.show_file(answer_path)
raise Exception('Time Limit Exceeded')
elif exec_code != 0:
self.show_result('Runtime Error',
input_path, time_duration, 'magenta')
self.show_file(input_path)
self.show_file(answer_path)
raise Exception('Runtime Error')
elif not diff.diff(self.get_path(answer_path),
self.get_path(output_path)):
self.show_result('Wrong Answer',
input_path, time_duration, 'red')
self.show_file(input_path)
self.show_file(answer_path)
self.show_file(output_path)
raise Exception('Wrong Answer')
self.show_result('OK', input_path, time_duration, 'green')
return time_duration
def push():
print 'Pushing as %s' % conf['username']
w = wiki.Wiki(conf['url'], conf['username'], conf['password'])
for (new_file, d) in diff.diff():
m = DIFF_FILE_RE.match(d[0])
if not m:
print 'Failed to parse diff for %s' % d[0]
continue
tempdir = tempfile.mkdtemp()
title = m.group(1)
new_file_msg = ''
if new_file:
new_file_msg = ', page is new'
print
print ('Editted file: %s (resolving in %s%s)'
%(title, tempdir, new_file_msg))
try:
with open(os.path.join(tempdir, title), 'w') as f:
if not new_file:
f.write(w.get_page(title).encode('ascii', 'replace'))
with open(os.path.join(tempdir, '.patch'), 'w') as f:
f.write(''.join(d))
cmd = 'cd %s; patch < .patch' % tempdir
(out, exit) = utils.execute(cmd)
if exit != 0:
print ' %s' % ' '.join(out)
continue
with open(os.path.join(tempdir, title), 'r') as f:
data = f.read()
w.post_page(title.replace('!slash!', '/'), data, minor=False,
bot=False)
finally:
pass
shutil.rmtree(tempdir)
def push():
print "Pushing as %s" % conf["username"]
w = wiki.Wiki(conf["url"], conf["username"], conf["password"])
for (new_file, d) in diff.diff():
m = DIFF_FILE_RE.match(d[0])
if not m:
print "Failed to parse diff for %s" % d[0]
continue
tempdir = tempfile.mkdtemp()
title = m.group(1)
new_file_msg = ""
if new_file:
new_file_msg = ", page is new"
print
print ("Editted file: %s (resolving in %s%s)" % (title, tempdir, new_file_msg))
try:
with open(os.path.join(tempdir, title), "w") as f:
if not new_file:
f.write(w.get_page(title).encode("ascii", "replace"))
with open(os.path.join(tempdir, ".patch"), "w") as f:
f.write("".join(d))
cmd = "cd %s; patch < .patch" % tempdir
(out, exit) = utils.execute(cmd)
if exit != 0:
print " %s" % " ".join(out)
continue
with open(os.path.join(tempdir, title), "r") as f:
data = f.read()
w.post_page(title.replace("!slash!", "/"), data, minor=False, bot=False)
finally:
pass
shutil.rmtree(tempdir)
def para_diff(actual, target, junk=[]):
""" Diff for paragraphs. Returns a list of phrases / operations to do to
translate the paragraphs in 'actual' to the paragraphs in 'target'.
Allowed phrases / operations are: (1) Split paragraph, (2) Merge paragraph,
(3) Rearrange phrase, (4) Insert phrase, (5) Delete phrase and (6) common
phrases (phrases which are common to actual and target).
Let us introduce a simple example to use throughout the documentation:
actual target
The quick fox jumps The quick fox jumps
over the lazy dog. over the lazy dog.
Pack my box with an
Pack my box with an thirty dozen liquor
twenty dozen liquor jugs.
jugs.
The actual text consists of 2 paragraphs, target consists of only 1
paragraph. Further thex differ in the words twenty <-> thirty.
Actually, 'actual' and 'target' have to be (arbitrarily nested) lists of
words. In our case, both lists have to be once nested lists of words of
paragraphs:
actual = [['The', 'quick', 'fox' ...], ['Pack', 'my', ...]]
target = [['The', 'quick', 'fox' ...]]
"""
# Flatten both lists of words to be able to do a word-based diff.
# 'actual_flatten' and 'target_flatten' are now flat lists of tuples. Each
# tuple (<word>, <flat_pos>, <pos_stack>) consists of:
# <word> : The word
# <flat_pos> : The pos of word in flat representation of original list.
# <pos_stack> : The position stack as list. The i-th element denotes the
# position of the word in the original list at level i.
# actual_flatten = [('The', 0, [0, 0]), ..., ('Pack', 9, [1, 0]), ...]
# target_flatten = [('The', 0, [0, 0]), ..., ('Pack', 9, [0, 9]), ...]
actual_flatten = flatten(actual)
target_flatten = flatten(target)
# Do a word-based diff on 'actual_flatten' and 'target_flatten'.
# The result is a list of diff.DiffCommonPhrase and diff.DiffReplacePhrase
# objects denoting the operations to perform to transform actual_flatten
# into target_flatten.
#
# [= [('The', 0, [0, 0]), ('quick', 1, [0, 1]), ('fox', 2, [0, 2])],
# [('The', 0, [0, 0]), ('quick', 1, [0, 1]), ('fox', 2, [0, 2])]]
# denotes a DiffCommonPhrase consisting of the related elements in
# actual_flatten and target_flatten. It implies that "The quick brown fox"
# occurs in both texts.
#
# [/ [('twenty', 13, [1, 5])], [('thirty', 13, [0, 13])]]
# denotes a DiffReplacePhrase consisting of the related elements in
# actual_flatten and target_flatten. It implies that we have to replace
# "twenty" in actual by "thirty" in target.
# One of the element lists in a DiffReplacePhrase may be empty, denoting
# either an insertion or a deletion.
# NOTE: The diff result does *not* respect any paragraph boundaries. Diff
# considers both list as an sequence of words. For example, a
# diff.DiffCommonPhrase may extend one or more paragraph boundaries.
diff_phrases = diff.diff(actual_flatten, target_flatten)
# Assume, that there is a phrase that occur in both list, but its order in
# actual doesn't correspond to the order in target, for example:
#
# The quick fox jumps Pack my box with an
# over the lazy dog. twenty dozen liquor
# jugs.
# Pack my box with an
# twenty dozen liquor The quick fox jumps
# jugs. over the lazy dog.
#
# The phrase "The quick fox ..." is rearranged in both texts. Diff won't
# find such a rearrangement. Instead it would state to delete the phrase
# from actual and to insert it in target.
# Try to find such phrases and to rearrange them.
rearranged_phrases = rearr.rearrange(diff_phrases, junk)
# Because diff doesn't know any paragraph boundaries (see above), we have
# to divide the phrases to get phrases per paragraph.
return divide_phrases_per_para(rearranged_phrases, junk)
import matplotlib.pyplot as plt
from macd import macd
from diff import diff
f, axarr = plt.subplots(5, 1)
Nwindow=500
h_diff_1 = macd(2, 4, Nwindow)
h_diff_2 = macd(5, 10, Nwindow)
h_diff_3 = macd(10, 20, Nwindow)
h_diff_4 = macd(20, 40, Nwindow)
h_diff_5 = macd(50, 100, Nwindow)
h_diff_11 = diff(2, 4, Nwindow)
h_diff_22 = diff(5, 10, Nwindow)
h_diff_33 = diff(10, 20, Nwindow)
h_diff_44 = diff(20, 40, Nwindow)
h_diff_55 = diff(50, 100, Nwindow)
with open('jpm_trades.csv', 'r') as trades_csv:
trades = np.array(list(csv.reader(trades_csv))[1:]).astype('double')
prices = trades[:,1]
log_prices = np.log(prices)
candidate1 = np.convolve(log_prices-log_prices[0], h_diff_1)
y1 = candidate1[:len(prices)]
axarr[0].plot(y1)
candidate11 = np.convolve(log_prices-log_prices[0], h_diff_11)
y11 = candidate11[:len(prices)]
#!/usr/bin/anaconda3/bin/python
import csv
import numpy as np
import matplotlib.pyplot as plt
from diff import diff
f, axarr = plt.subplots(5, 1)
Nwindow=500
h_diff_1 = diff(2, 4, Nwindow)
h_diff_2 = diff(5, 10, Nwindow)
h_diff_3 = diff(10, 20, Nwindow)
h_diff_4 = diff(20, 40, Nwindow)
h_diff_5 = diff(50, 100, Nwindow)
with open('jpm_trades.csv', 'r') as trades_csv:
trades = np.array(list(csv.reader(trades_csv))[1:]).astype('double')
prices = trades[:,1]
log_prices = np.log(prices)
candidate1 = np.convolve(log_prices-log_prices[0], h_diff_1)
y1 = candidate1[:len(prices)]
axarr[0].plot(y1)
axarr[0].set_title('Macd-M1 of log px series: Neff+ = 2')
candidate2 = np.convolve(log_prices-log_prices[0], h_diff_2)
y2 = candidate2[:len(prices)]
axarr[1].plot(y2)
def main():
"""
Get command line arguments and invoke the command line functionality.
"""
filters = []
parser = argparse.ArgumentParser()
parser.add_argument('replay_file', help='.SC2Replay file to load',
nargs='?')
parser.add_argument("--gameevents", help="print game events",
action="store_true")
parser.add_argument("--messageevents", help="print message events",
action="store_true")
parser.add_argument("--trackerevents", help="print tracker events",
action="store_true")
parser.add_argument("--attributeevents", help="print attributes events",
action="store_true")
parser.add_argument("--attributeparse", help="parse attributes events",
action="store_true")
parser.add_argument("--header", help="print protocol header",
action="store_true")
parser.add_argument("--metadata", help="print game metadata",
action="store_true")
parser.add_argument("--details", help="print protocol details",
action="store_true")
parser.add_argument("--details_backup", help="print protocol anoynmized details",
action="store_true")
parser.add_argument("--initdata", help="print protocol initdata",
action="store_true")
parser.add_argument("--all", help="print all data",
action="store_true")
parser.add_argument("--quiet", help="disable printing",
action="store_true")
parser.add_argument("--stats", help="print stats",
action="store_true")
parser.add_argument("--diff", help="diff two protocols",
default=None,
action="store")
parser.add_argument("--versions", help="show all protocol versions",
action="store_true")
parser.add_argument("--types", help="show type information in event output",
action="store_true")
parser.add_argument("--json", help="print output as json",
action="store_true")
parser.add_argument("--ndjson", help="print output as ndjson (newline delimited)",
action="store_true")
parser.add_argument("--profile", help="Whether to profile or not",
action="store_true")
args = parser.parse_args()
if args.profile:
pr = cProfile.Profile()
pr.enable()
# TODO: clean up the command line arguments to allow cleaner sub-command
# style commands
# List all protocol versions
if args.versions:
files = versions.list_all()
pattern = re.compile('^protocol([0-9]+).py$')
captured = []
for f in files:
captured.append(pattern.match(f).group(1))
if len(captured) == 8:
print >> sys.stdout, captured[0:8]
captured = []
print >> sys.stdout, captured
return
# Diff two protocols
if args.diff and args.diff is not None:
version_list = args.diff.split(',')
if len(version_list) < 2:
print >> sys.stderr, "--diff requires two versions separated by comma e.g. --diff=1,2"
sys.exit(1)
diff.diff(version_list[0], version_list[1])
return
# Check/test the replay file
if args.replay_file is None:
print >> sys.stderr, ".S2Replay file not specified"
sys.exit(1)
archive = mpyq.MPQArchive(args.replay_file)
filters = []
if args.json:
filters.insert(0, JSONOutputFilter(sys.stdout))
elif args.ndjson:
filters.insert(0, NDJSONOutputFilter(sys.stdout))
elif not args.quiet:
filters.insert(0, PrettyPrintFilter(sys.stdout))
if args.types:
filters.insert(0, TypeDumpFilter())
if args.stats:
filters.insert(0, StatCollectionFilter())
def process_event(event):
for f in filters:
event = f.process(event)
# Read the protocol header, this can be read with any protocol
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
if args.header:
process_event(header)
# The header's baseBuild determines which protocol to use
baseBuild = header['m_version']['m_baseBuild']
try:
protocol = versions.build(baseBuild)
except Exception, e:
print >> sys.stderr, 'Unsupported base build: {0} ({1})'.format(baseBuild, str(e))
sys.exit(1)
def test_make_png_diff_img(self):
# these images are the same, but it shouldn't matter for this test
diff(IMG4, IMG4, diff_img_file=TEST_PNG_OUT)
self.assertTrue(os.path.exists(TEST_PNG_OUT))
os.remove(TEST_PNG_OUT)
def test_make_jpg_diff_img(self):
diff(IMG1, IMG2, diff_img_file=TEST_JPG_OUT)
self.assertTrue(os.path.exists(TEST_JPG_OUT))
os.remove(TEST_JPG_OUT)
def test_bw_image_ratio(self):
ratio = diff(BLACK_IMG, WHITE_IMG, delete_diff_file=True)
self.assertEqual(ratio, 1.0)
from ema import ema
from macd import macd
from diff import diff
Nwindow = 400
Neff_pos = 20
Neff_neg = 40
Neff = 20
Nbox = 20
h_step = step(Nwindow)
h_box = box(Nbox, Nwindow)
h_delta = delta(Neff_pos, Nwindow)
h_ema = ema(Neff_pos, Nwindow)
h_macd = macd(Neff_pos, Neff_neg, Nwindow)
h_diff = diff(Neff_pos, Neff_neg, Nwindow)
f, axarr = plt.subplots(3, 2)
axarr[0, 0].plot(h_step)
axarr[0, 0].set_title("step")
axarr[0, 1].plot(h_box)
axarr[0, 1].set_title("box")
axarr[1, 0].plot(h_delta)
axarr[1, 0].set_title("delta")
axarr[1, 1].plot(h_diff)
axarr[1, 1].set_title("diff - macd m1")
def main(archive_dir, debug = 0):
if not os.path.isdir(archive_dir):
print "archive_dir "+archive_dir+" not found"
return -1
logger.debug("DEBUG = "+str(debug))
fpns_to_sync = [x for x in [x.strip() for x in open(os.path.join(utils.getScriptPath(), "fpns_to_sync.txt")).read().\
split("\n")] if x[0] != "#"] # there!
logger.debug("WILL SYNC "+",".join(fpns_to_sync))
if len(fpns_to_sync) == 0:
logger.error("Nothing to sync")
return -1
logger.debug("connecting to sp")
spsoap = spsoapclient.spsoapclient()
sp_list_map = spsoap.get_lists_as_map()
def _check_lists():
bad = False
for ed in fpns_to_sync:
try:
sp_list_map[ed]
except KeyError:
logger.error("List to sync:%s not in SP"%ed)
bad = True
return bad
if _check_lists():
return -1
rmap = spsoap.get_recips_as_map(os.path.join(tempdir, utils.temp_name("sp_export.csv")))
logger.debug("rmap len %d"%len(rmap))
now = datetime.datetime.now()
nowfile = os.path.join(archive_dir, "now.csv")
prevfile = os.path.join(archive_dir, "prev.csv")
if debug != 1:
if not os.path.isfile(nowfile):
logger.debug("there is no now file - wtf do we do")
else:
if os.path.isfile(prevfile):
n = os.path.join(archive_dir,
"prev_%04d%02d%02d.csv"%(now.year, now.month, now.day))
if os.path.isfile(n):
os.remove(n)
os.rename(prevfile, n)
os.rename(nowfile, prevfile)
__dumpSF(nowfile)
rc = spsfsync.main(nowfile, rmap, None, send_emails_to)
import diff
adds, deletes = diff.diff(nowfile, prevfile)
if len(adds) == 0 and len(deletes) == 0:
logger.debug("Nothing to do")
return 0
good_adds = []
bad_adds = []
good_dels = []
bad_dels = []
for ed in fpns_to_sync:
listid = sp_list_map[ed]
logger.debug("DOING %s - %d"%(ed, listid))
if ed in adds:
for a in adds[ed]:
logger.debug("add %s to %s id %d"%(a, ed, listid))
if a in rmap:
recip_id = rmap[a][0]
else:
try:
recip_id = spsoap.create_recip(a)
except spsoapclient.spsoapclient_exception, e:
bad_adds.append("%s to %s failed with %s"%(a,ed,e.what))
continue
rc = spsoap.add_recip_to_list(int(listid), int(recip_id))
good_adds.append("%s to %s"%(a, ed))
logger.debug("DOING dels %d"%(len(deletes)))
if ed in deletes:
for a in deletes[ed]:
logger.debug("delete %s from %s id %d"%(a, ed, listid))
if a in rmap:
recip_id = rmap[a][0]
mailing_id = rmap[a][2]
rc = spsoap.delete_recip_from_list(int(listid), int(recip_id), int(mailing_id))
good_dels.append("%s from %s"%(a, ed))
else:
bad_dels.append("%s unsub from %s is not in recip list of SP"%(a, ed))
from diff import Diff as diff
def testNegative(self):
"""Diff should never output negative values"""
result = diff.diff(0, 'test')
self.assertFalse(result < 0)
def main(source, dest, runtype):
# runtype 1 - the one run at midnight
# runtype 2 - the final day's run
global logger
logger = utils.createLogger("runner")
if not os.path.isdir(source):
print "invalid directory "+source
sys.exit(1)
if not prepareDestDir(dest):
sys.exit(1)
import datetime
today = datetime.datetime(2012, 12, 21).now()
pfile = os.path.join(source, "prev_%s.csv"%runtype)
nfile = os.path.join(source, "now_%s.csv"%runtype)
backup(source, today, runtype)
# abort if there is no prev file
if not os.path.isfile(pfile):
logger.error("There is no previous file "+pfile)
sendMail(None, "**** Stormpost SF Error", "There is no previous file "+pfile)
sys.exit(1)
import p2
bx = utils.getSFSiteCred()
p2.main(int(runtype), nfile, bx[0], bx[1], bx[2])
if not os.path.isfile(nfile):
restorePrev(source, runtype)
return
import diff
adds, deletes = diff.diff(nfile, pfile)
print adds, deletes
tosend = []
jlt_to_old = {
"JLT Pharma":"Pharma Global",
"JLT Constr":"Constr Global",
"JLT Insur":"Insur Global",
"JLT Bank":"Bank Global",
"JLT Telecom":"Telecom Global",
"JLT Asset & Wealth":"Asset & Wealth Global",
"JLT Lawyers":"Lawyers Global",
"JLT Media":"Media Global"}
global_to_new = {
"Pharma Global":"Pharma Global Advisen",
"Constr Global":"Constr Global Advisen",
"Insur Global":"Insur Global Advisen",
"Bank Global":"Bank Global Advisen",
"Telecom Global":"Telecom Global Advisen",
"Asset & Wealth Global":"Asset & Wealth Global Advisen",
"Lawyers Global":"Lawyers Global Advisen",
"Media Global":"Media Global Advisen"}
def createSPFiles(which, decor):
import string
for w in which.keys():
if w in jlt_to_old.keys(): k = jlt_to_old[w]
else:
if w in global_to_new: k = global_to_new[w]
else:
k = w
if decor == "":
fn = k.replace(" ", "_").replace("/", "_") + "_%04d_%02d_%02d.txt"%(today.year, today.month, today.day)
else:
fn = k.replace(" ", "_").replace("/", "_") + "_%s_%04d_%02d_%02d.txt"%(decor,today.year, today.month, today.day)
fn = os.path.join(dest, fn)
tosend.append(fn)
op = open(fn, "w").write(string.join(which[w], "\n"))
createSPFiles(adds, "")
createSPFiles(deletes, "remove")
logger.debug("Files to send "+str(tosend))
sys.exit(0)
float_type = np.float32
#float_type = np.float64
nloop = 100;
itot = 384;
jtot = 384;
ktot = 384;
ncells = itot*jtot*ktot;
at = np.zeros((ktot, jtot, itot), dtype=float_type)
index = np.arange(ncells, dtype=float_type)
a = (index/(index+1))**2
del(index)
a.shape = (ktot, jtot, itot)
# Check results
diff.diff(at, a, 0.1, 0.1, 0.1, 0.1)
print("at={0}".format(at.flatten()[itot*jtot+itot+itot//2]))
# Time the loop
start = timer()
for i in range(nloop):
diff.diff(at, a, 0.1, 0.1, 0.1, 0.1)
end = timer()
print("Time/iter: {0} s ({1} iters)".format((end-start)/nloop, nloop))
def test_mario_ratio(self):
ratio = diff(IMG1, IMG2, delete_diff_file=True)
self.assertEqual(ratio, 0.007319618135968298)
#%% from `test_render.py`, `test_upstream(tmpdir,wavedromdir,file)`
from IPython.display import SVG, display
test_name = splitext(basename(file))[0]
f_out_js = "{}/{}_js.svg".format(tmpdir, test_name)
f_out_py = "{}/{}_py.svg".format(tmpdir, test_name)
subprocess.check_call("node {}/bin/cli.js -i {} > {}".format(wavedromdir, file, f_out_js), shell=True)
wavedrom.render_file(file, f_out_py, strict_js_features=True)
display('wavedrom:')
display(SVG(f_out_js))
display('wavedrompy:')
display(SVG(f_out_py))
unknown = diff(f_out_js, f_out_py)
if len(unknown) > 0:
msg = "{} mismatch(es)\n".format(len(unknown))
msg += "js file: {}\npy file: {}\n".format(f_out_js, f_out_py)
msg += "\n".join([str(action) for action in unknown])
#pytest.fail(msg)
display(msg)
raster_difference = diff_raster(f_out_js, f_out_py)
#%% [markdown]
# # Analyze Raster Difference
# This cell does some analysis on the difference between the two images generarted by rasterizing the SVG output by *wavedrom* and *wavedrompy*. The cell saves several channel-specific differences to images so the differences can be visualized.
#
# The process should work with differences in all four channels of an RGBA image. It will not work with L (grayscale) images. This has only been tested with RGBA images that have differences in either:
def test_delete_diff_img(self):
if os.path.exists(TEST_JPG_OUT):
os.remove(TEST_JPG_OUT)
diff(IMG1, IMG2, delete_diff_file=True, diff_img_file=TEST_JPG_OUT)
self.assertFalse(os.path.exists(TEST_JPG_OUT))
# Get SMA data
SMA_file = '/home/pi/smadata/%d/MyPlant-Spot-SingleLine.csv' % datetime.datetime.now(
).year
try:
with open(SMA_file, 'r') as fp:
reader = csv.reader(fp, delimiter=';')
SMA_data = next(reader)
for key, value in enumerate(SMA_data):
SMA_data[key] = SMA_data[key].replace(',', '.')
except FileNotFoundError:
nebula_client.error('SMA data file not found! [%s]' % SMA_file)
# Prepare Nebula payload
E_PV = float(SMA_data[23]) # Solar Energy Production [kWh]
E_cons = E_PV + E_net # Local Energy Consumption [kWh]
P_prod = diff(E_PV, 'E_PV') * (
60 / 5) * 1000 # 5 minute average Solar Energy Production [W]
P_net = diff(E_net, 'E_net') * (
60 / 5) * 1000 # 5-minute average Net ENergy Consumption [W]
P_cons = P_prod + P_net
payload = {
'32': P_net, # Net Power Consumption [W]
'30': E_net, # Net Energy Consumption [kWh]
'43': P_cons, # Local Power Consumption [W]
'33': P_prod, # Solar Power Production [W]
'34': SMA_data[22], # Solar Energy Production Today [kWh]
'35': SMA_data[24], # Inverter Frequency [Hz]
'36': SMA_data[30], # Inverter Temperature [C]
'37': E_PV, # Solar Energy Production Total [kWh]
'42': psutil.cpu_percent(), # Current system-wide CPU utilization [%]
def test_different_modes(self):
with self.assertRaises(ValueError):
diff(IMG3, IMG4)
def differ(path_a, path_b):
tree_a = ET.parse(path_a)
root_a = tree_a.getroot()
tree_b = ET.parse(path_b)
root_b = tree_b.getroot()
return diff.diff(root_a, root_b)
def test_diff_filename_without_extension_saves_as_png(self):
diff(IMG4, IMG4, diff_img_file=TEST_NO_EXT_OUT)
im = PIL.Image.open(TEST_NO_EXT_OUT)
self.assertEqual(im.format, 'PNG')
os.remove(TEST_NO_EXT_OUT)
import add import diff import mult import division c = add.sum(5, 6) print(c) c = diff.diff(6, 5) print(c) c = mult.mult(5, 2) print(c) c = division.divison_numbers(10, 2) print(c)
def para_diff(actual, target, junk=[]):
""" Finds the differences between the two given lists of paragraphs. """
# 'actual' and 'target' may be arbitrarily nested list of words.
# In our case, both lists are once nested list of words of paragraphs.
# example: 'actual_paras' = [['words', 'of', 'first', 'paragraph], [...]]
# Flatten the list of words to be able to do a word-based diff.
actual_flatten = flatten(actual)
target_flatten = flatten(target)
# 'actual_flatten' and 'target_flatten' are now flat lists of tuples. Each
# tuple (<word>, <flat_pos>, <pos_stack>) consists of:
# <word> : The word
# <flat_pos> : The pos of word in flat representation of original list.
# <pos_stack> : The position stack as list. The i-th element denotes the
# position of the word in the original list at level i.
# example: flatten([['foo', 'bar'], ['baz']])
# = [('foo', 0, [0, 0]), ('bar', 1, [0, 1]), ('baz', 2, [1, 0])]
# Do a word-based diff on 'actual_flatten' and 'target_flatten'.
# The result is a list of diff.Common and diff.Replace objects denoting
# the operations to perform to transform actual_flatten into target_flatten.
# Both objects contain the related elements in 'actual_flatten' and
# 'target_flatten'
#
# examples:
#
# (= [('foo', 0, [0, 0]), ('bar', 1, [0, 1]), ('baz', 2, [1, 0])],
# [('foo', 0, [0, 0]), ('bar', 1, [0, 1]), ('baz', 2, [1, 0])])
# denotes a common object including the related elements in actual_flatten
# and the elements in target_flatten. It implies that "foo bar baz" occurs
# in both lists.
#
# (/ [('foo', 0, [0, 0]), ('bar', 1, [0, 1]), ('baz', 2, [1, 0])],
# [('doo', 0, [0, 0])])
# denotes a replace object including the related elements in actual_flatten
# and the elements in target_flatten. It implies that "foo bar baz" in
# 'actual' is replaced by "doo" in 'target'.
#
# One of the element lists in diff.Replace objects my be empty, denoting
# either a insert or an deletion.
diff_result = diff.diff(actual_flatten, target_flatten)
# There could be phrases that occur in both, 'actual_flatten' and
# 'target_flatten' but their order doesn't correspond. Try to identify and
# rearrange such phrases.
rearrange_result = rearr.rearrange(diff_result, junk)
# The rearrange result is now a flat list of diff.Common, diff.Replace and
# rearr.Rearrange objects and doesn't meet any paragraph structures.
# So we need to split (and merge) the objects now to get operations per
# paragraph.
para_result = []
# Keep track of the previous actual item and the previous target item to
# be able to decide where to split the objects.
prev_item_actual = None
prev_item_target = None
for item in rearrange_result:
if isinstance(item, diff.Common):
item = Commons(item, prev_item_actual, prev_item_target)
elif isinstance(item, rearr.Rearrange):
item = Rearranges(item, prev_item_actual, prev_item_target)
elif isinstance(item, diff.Replace):
item = Replaces(item, prev_item_actual, prev_item_target)
# TODO: Obtain the previous actual item and the previous target item.
if item and item.phrases:
xxx = [x for x in item.phrases if not isinstance(x, Delete) and not util.ignore_phrase(x, junk)]
if xxx:
last_item = xxx[-1]
if last_item.items_actual:
prev_item_actual = last_item.items_actual[-1]
yyy = [y for y in item.phrases if not isinstance(y, Insert) and not util.ignore_phrase(y, junk)]
if yyy:
last_item = yyy[-1]
if last_item.items_target:
prev_item_target = last_item.items_target[-1]
para_result.append(item)
# return merge(para_result)
return para_result
import diff
import os
nfile = "/tmp/test.csv"
pfile = "/tmp/split.csv"
adds, deletes = diff.diff(nfile, pfile)
print adds, deletes
tosend = []
dest = "/tmp/rrx"
import datetime
today = datetime.datetime(2012, 12, 21).now()
def createSPFiles(which, decor):
import string
for k in which.keys():
#fixme: i think we are losing the EU changes between 1 and 2 runs
# if (runtype == "1" and k in fpn_type_1) or \
# (runtype == "2" and k not in fpn_type_1 and k not in fpn_type_2) or\
# (runtype == "3" and k in fpn_type_2):
inc = 1
if inc == 1:
if decor == "":
fn = k.replace(" ", "_").replace("/", "_") + "_%04d_%02d_%02d.txt"%(today.year, today.month, today.day)
else:
fn = k.replace(" ", "_").replace("/", "_") + "_%s_%04d_%02d_%02d.txt"%(decor,today.year, today.month, today.day)
fn = os.path.join(dest, fn)
tosend.append(fn)
op = open(fn, "w").write(string.join(which[k], "\n"))
createSPFiles(adds, "")
createSPFiles(deletes, "remove")
