def __init__(self, dimension, kwargs):
self.values = kwargs
if not self.values.has_key('from') and not self.values.has_key('to'):
fail("Bad inequality filter")
self.dimension = dimension
self.dimensionTable = dimension.table
self.dimensionField = dimension.field
def writeData(self, events_data):
self.report()
signals = {}
# Add all the signals
for signal in self.signal_elements:
try:
signal_value = getattr(self, signal)
if signal_value is not None:
signals[signal] = signal_value
except:
fail("Sensor: '" + self.name + "' does not contain member: " +
signal)
# For all events linked with this signal, run isTriggered() and add to response
events_triggered = {}
for event in events_data:
if self.name in event.attached_sensors:
try:
if event.isTriggered():
events_triggered[event.PARAM__name] = event.getDict()
except Exception as e:
print("Error while running isTriggered() for event: " +
str(event.PARAM__info) + "\n" + str(e))
response = {}
if len(signals) > 0:
response['signals'] = signals
if len(events_triggered) > 0:
response['events'] = events_triggered
self.afterReport()
return response
def do_login_admin(browser):
common.info("Processing login..")
function_result = True
# go to the admin login page and wait for page to be loaded
browser.get("http://localhost/litecart/admin/")
WebDriverWait(browser, 10).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "button[value=Login]")))
# enter credentials
login_field = browser.find_element_by_name("username")
login_field.clear()
login_field.send_keys("admin")
password_field = browser.find_element_by_name("password")
password_field.clear()
password_field.send_keys("admin")
# press [Login] button and wait for next page to be loaded
browser.find_element_by_css_selector('button[name="login"]').click()
WebDriverWait(browser, 10).until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "li#widget-discussions")))
if (browser.find_element_by_css_selector("li#widget-discussions")):
function_result = True
common.success("Login successful")
else:
function_result = False
common.fail("Cannot login to admin page")
return function_result
def build_year_list(in_dir):
year_lists = []
for subdir, dirs, files in os.walk(in_dir):
for file in files:
if file[0] != '.' and file[-5:] == '.json':
with open(in_dir + "/" + file, 'r',
encoding='utf8') as in_file:
jsondata = json.load(in_file)
year_lists.append(sorted(jsondata['year list']))
elif file[0] != '.' and file[-4] == '.csv':
with open(in_dir + "/" + file, 'r',
encoding='utf8') as in_file:
read_csv = csv.reader(in_file, delimiter=',')
for row in read_csv:
if row[0] == "word" and row[1] == "tf-idf avg":
years = row[7].split()
year_list = sorted([int(year) for year in years])
year_lists.append(year_list)
for year_list in year_lists:
if year_list != year_lists[0]:
common.fail(
"One of your files has a different year list from the others."
+
" Please make sure all your files contain the same year lists."
)
# just return first list in year_lists if they're all the same
return year_lists[0]
def read_one_sequence(defects, get_line, _pushback, fname):
t = get_line()
if t == None:
return None
(when, msg) = t
if not START_INTERFERING_RE.match(msg):
common.fail("Lost sequencing at %s in %s" % (msg, fname))
start_time = when
expected_start = when
r = []
key = None
while True:
(when, msg) = get_line()
if msg == "stop interfering CFG":
end_time = when
(_, msg) = get_line()
w = msg.split()
if w[:2] != ["high", "water:"]:
common.fail("Lost sequencing 2 at %s" % msg)
break
w = msg.split()
if msg in [early_out, satisfiable, unsatisfiable, ic_atomic_false]:
r.append(msg)
continue
assert w[0] == "start"
start = when
key = " ".join(w[1:])
assert key in bubble_keys
if not defects.has_key(key):
defects[key] = 0
defects[key] += start - expected_start
t = get_line()
if t == None:
return None
(when, msg) = t
rr = {"key": key, "duration": when - start}
if msg == "timeout":
rr["failed"] = "timeout"
elif msg == "out of memory":
rr["failed"] = "oom"
else:
assert msg.split()[0] == "stop" and " ".join(msg.split()[1:]) == key
rr["failed"] = None
expected_start = when
r.append(rr)
if rr["failed"]:
end_time = when
break
# Check for leftovers
t = get_line()
assert t == None
if not defects.has_key("leftover"):
defects["leftover"] = 0
defects["leftover"] += end_time - expected_start
if key != None:
r.append({"key": key, "duration": end_time - expected_start, "failed": None})
return (end_time - start_time, r)
def remove(args, keep_images=[], keep_containers=[]):
if not common.args_check(args, 1):
common.fail(
"Remove all containers and images associated with an image.\n\nUsage:\n\n jockler remove IMAGENAME"
)
imagename = args[0]
allimages = listing.get_image_list_for(imagename)
allcontainers = listing.get_container_list_for(imagename)
for k in set(keep_containers):
try_remove(k, allcontainers)
for k in set(keep_images):
try_remove(k, allimages)
if not confirm_removal(allcontainers, allimages, keep_containers,
keep_images):
common.fail("ABORTED")
# Remove the image tag
run.call(["docker", "rmi", imagename])
run.call(["docker", "rm"] + allcontainers, stdout=sys.stdout, silent=True)
run.call(["docker", "image", "rm"] + allimages,
stdout=sys.stdout,
silent=True)
store.cleanup_images(imagename)
def stop_containers(imagename):
containernames = get_running_containers(imagename)
if len(containernames) > 0:
code, sout, serr = run.call( ["docker", "stop"] + containernames )
if code > 0:
common.fail("Error stopping container(s) !\n%s"%(sout))
code, sout, serr = run.call( ["docker", "update", "--restart=no"] + containernames )
def list_container_volumes(imagename, instance):
if not instance in ["stable", "last"]:
common.fail("Invalid instance [%s] - use last or stable" % instance)
containername = store.read_data(instance, imagename)
if not containername:
common.fail("No candidate [%s] for image [%s]" % (instance, imagename))
print(os.linesep.join(mountdata_of(containername, "Name")))
def get_image_of(containername):
res,sout,serr = run.call(["docker", "ps", "-a", "--format", "{{.Image}}"] + ps_filter(containername), silent=True)
images = sout.strip().split("\n")
common.remove_empty_strings(images)
if len(images) != 1:
common.fail("Could not find single image for %s"%containername)
return images[0]
def listing(args):
if not common.args_check(args, 2):
common.fail(helpstr)
category = args[0]
imagename = common.item(args, 1)
if imagename == ".all" or imagename == None:
list_all(category)
else:
list_on_image(imagename, category)
def stable(args):
if not common.args_check(args, 1):
common.fail(helpstr)
imagename = args[0]
if common.args_check(args, 2) and imagename == ":show":
imagename = args[1]
print_stable_container(imagename)
else:
mark_stable_container(imagename)
def mark_stable_container(imagename):
containernames = container.get_running_containers(imagename)
if len(containernames) != 1:
print(containernames)
common.fail("Could not identify single container to mark as stable")
containername = containernames[0]
store.write_data("stable", imagename, containername)
print("Marked %s as stable. Start it again using 'jockler start stable %s'"%(containername, imagename))
def read_one_sequence(defects):
try:
(when, msg) = common.get_line()
except StopIteration:
return None
w = msg.split()
if w[:-1] != ["start", "build", "enforcer"]:
common.fail("Lost sequencing at %s" % msg)
start_time = when
expected_start = when
nr_produced = 0
r = []
key = None
while True:
(when, msg) = common.get_line()
if msg == "stop build enforcer":
end_time = when
if key != None:
r.append({"key": key, "duration": end_time - expected_start, "failed": None})
break
w = msg.split()
assert w[0] == "start"
start = when
key = " ".join(w[1:])
assert key in bubble_keys
if not defects.has_key(key):
defects[key] = 0
assert start >= expected_start
defects[key] += start - expected_start
(when, msg) = common.get_line()
assert when > start
rr = {"key": key, "duration": when - start}
r.append(rr)
if msg == "timeout":
rr["failed"] = "timeout"
end_time = when
break
elif msg == "out of memory":
rr["failed"] = "oom"
end_time = when
break
else:
w = msg.split()
assert msg.split()[0] == "stop" and " ".join(msg.split()[1:]) == key
rr["failed"] = None
expected_start = when
if not defects.has_key("leftover"):
defects["leftover"] = 0
assert end_time > expected_start
defects["leftover"] += end_time - expected_start
return ((end_time - start_time, r), nr_produced)
def parseConfigFile(filename="./config.xml"):
dom = parse(filename)
rootElement = dom.documentElement
if rootElement.tagName != "storages":
fail("Bad xml!")
storageElements = rootElement.getElementsByTagName("storage")
storages = {}
for element in storageElements:
storage = Storage(element)
storages[storage.getId()] = storage
return storages
def datatest():
i = str(os.urandom(10)).replace('/','|')
def recive(x,y):
tester[x] = y
subscribe(i,recive)
x =str(os.urandom(100))
postMessage(i,x)
time.sleep((random.random()/100)+0.05)
if not tester[i] == x:
fail('Message not properly recieved')
else:
pass
def getTable(self, dimensions, filters = []):
if len(dimensions) != 2:
fail("Bad dimensions size")
horizontalFilters = []
verticalFilters = []
for f in filters:
if f.dimension.name == self.dimensions[dimensions[0]].name:
horizontalFilters.append(f)
if f.dimension.name == self.dimensions[dimensions[1]].name:
verticalFilters.append(f)
horizontalNames = self.getNamesForOneDimension(self.dimensions[dimensions[0]], horizontalFilters)
verticalNames = self.getNamesForOneDimension(self.dimensions[dimensions[1]], verticalFilters)
connection = sqlite3.connect(self.path)
query = "select "
for dimensionName in dimensions:
dimension = self.dimensions[dimensionName]
query += dimension.table + "." + dimension.field + ", "
query += "sum(" + self.aggregatedValueField + ") "
query += " from " + self.factTable
query += self.joins
query += " where 1=1 "
for f in filters:
query += f.generateQueryString()
query += " group by "
for dimensionName in dimensions:
dimension = self.dimensions[dimensionName]
query += dimension.table + "." + dimension.field + ", "
query = query[:-2]
cursor = connection.execute(query)
resultDict = {}
for row in cursor:
d = resultDict.get(row[0], {})
d[row[1]] = row[2]
resultDict[row[0]] = d
ans = []
for h in horizontalNames:
last = []
for v in verticalNames:
if resultDict.has_key(h) and resultDict[h].has_key(v):
last.append(resultDict[h][v])
else:
last.append(0)
ans.append(last)
return horizontalNames, verticalNames, ans
def stop(args):
if not common.args_check(args, 1):
common.fail("Unknown sequence for stop: %s" % ' '.join(args))
imagename = args[0]
forcemode = False
if imagename == "-f" and common.item(args, 1, None) != None:
forcemode = True
if forcemode:
for containername in args[1:]:
force_stop(containername)
else:
stop_containers(imagename)
def do_volume_restore(imagename, systemname, archivefile, destination="."):
destination = os.path.abspath(destination)
archive_fullpath = os.path.sep.join([destination, archivefile])
volumes_array = options.extract_section(imagename, "-v", "volumes")
volume_mountpoints = options.extract_container_side(imagename, "volumes")
if not os.path.isfile(archive_fullpath):
common.fail("Could not find [%s]" % archive_fullpath)
if systemname == "linux":
res, sout, serr = do_linux_clear_data(volumes_array,
volume_mountpoints)
if res >= 0:
res, sout, serr = do_linux_volume_restore(volumes_array,
destination, archivefile)
elif systemname == "windows":
#TODO - need a windows version
common.fail("Not yet implemented")
else:
common.fail("Unkown system type [%s]" % systemname)
if res == 0:
print("Restore complete from %s" % archivefile)
else:
common.fail("Restore operation failed!")
def datatest():
i = str(os.urandom(10)).replace('/', '|')
def recive(x, y):
tester[x] = y
subscribe(i, recive)
x = str(os.urandom(100))
postMessage(i, x)
time.sleep((random.random() / 100) + 0.05)
if not tester[i] == x:
fail('Message not properly recieved')
else:
pass
def datatest2():
i = 'potty/'+str(os.urandom(10)).replace('/','|')
def recive(x,y):
tester[x] = y
subscribe('potty/',recive)
x =str(os.urandom(100))
postMessage(i,x)
time.sleep((random.random()/100)+0.05)
if not tester[i] == x:
fail('Message not properly recieved.')
else:
#print("hooray")
pass
def datatest2():
i = 'potty/' + str(os.urandom(10)).replace('/', '|')
def recive(x, y):
tester[x] = y
subscribe('potty/', recive)
x = str(os.urandom(100))
postMessage(i, x)
time.sleep((random.random() / 100) + 0.05)
if not tester[i] == x:
fail('Message not properly recieved.')
else:
#print("hooray")
pass
def select_from_dropdown_list(browser, data_to_set):
value_to_set = data_to_set["value"]
# if no value should be set then just return
if (not value_to_set):
return True
field_to_be_set = browser.find_element_by_css_selector(data_to_set["location"])
select_field = Select(field_to_be_set)
try:
WebDriverWait(browser, 15).until(EC.element_to_be_clickable((By.CSS_SELECTOR, data_to_set["location"])))
select_field.select_by_value(value_to_set)
if (field_to_be_set.get_property("value") == value_to_set):
common.info("Set %s = '%s': ok" % (data_to_set["location"], data_to_set["description"]))
return True
except:
common.fail("Set %s = '%s'" % (data_to_set["location"], data_to_set["description"]))
return False
def determine_index(data_type):
index = 0
if data_type == 'tf-idf avg':
index = 1
elif data_type == 'tf-idf max':
index = 2
elif data_type == 'tf-idf min':
index = 3
elif data_type == 'word frequency':
index = 4
elif data_type == 'average frequency':
index = 5
elif data_type == 'variance':
index = 6
else:
common.fail('You shouldn\'t be able to get here, congratulations!!')
return index
def start_container(imagename, containername, doattach=False):
stop_containers(imagename)
store.write_data("last", imagename, containername)
runmode = []
useexec = False
if doattach:
runmode.append("-i")
useexec = True
print("Starting %s"%containername)
code, sout, serr = run.call( ["docker", "start", containername]+runmode, stdout=sys.stdout, stderr=sys.stderr, useexec=useexec )
if code > 0 or not found_running_container(containername):
common.fail("Could not start container %s - try starting with 'attach' mode'\n%s"%(containername,sout))
code, sout, serr = run.call( ["docker", "update", "--restart=unless-stopped", containername])
def cleanup(args):
if not common.args_check(args, 1):
common.fail(
"Remove all old containers and images, except for last and stable.\n\nUsage:\n\n jockler cleanup IMAGENAME"
)
imagename = args[0]
keep_images = []
keep_containers = []
for instance in ["stable", "last"]:
container = store.read_data(instance, imagename)
if container:
keep_images.append(imagename)
keep_containers.append(container)
remove([imagename], keep_images, keep_containers)
def attach(args):
if not common.args_check(args, 1):
common.fail("Usage:\n jockler attach IMAGENAME [COMMAND]")
imagename = args[0]
shellname = "bash"
if common.args_check(args, 2):
shellname = args[1]
containernames = container.get_running_containers(imagename)
if len(containernames) != 1:
common.fail("Could not identify single container to attach to:\n%s" %
(str(containernames)))
command_tokens = ["docker", "exec", "-it", containernames[0], shellname]
os.execvp(command_tokens[0], command_tokens)
def __init__(self, element):
self.id = checkElementAttribute(element, "id")
self.path = checkElementAttribute(element, "path")
self.aggregatedValueName = checkElementAttribute(element, "aggregated_value_name")
self.factTable = checkElementAttribute(element, "fact_table")
self.aggregatedValueField = checkElementAttribute(element, "aggregated_value_field")
self.dimensions = {}
self.joins = ""
dimensionsHolder = element.getElementsByTagName("dimensions")
if len(dimensionsHolder) > 1:
fail("Too many dimensions elements!")
if len(dimensionsHolder) == 0 :
fail("No dimensions elements!")
dimensionsElements = dimensionsHolder[0].getElementsByTagName("dimension")
for element in dimensionsElements:
dimension = Dimension(element)
self.joins += " join " + dimension.table + " on " + dimension.factFk + "=" + dimension.table + ".id"
self.dimensions[dimension.name] = dimension
def start(args):
if len(args) >= 2:
instance = args[0]
imagename = args[1]
doattach = common.item(args, 2, False)
if doattach == "attach":
doattach=True
if not instance in ["new", "last", "stable"]:
common.fail("Incorrect instance name. Please use 'new', 'last', or 'stable' ")
if instance == "new":
containername = start_new_container(imagename, doattach)
else:
containername = store.read_data(instance, imagename)
if containername:
start_container(imagename, containername, doattach)
else:
common.fail("No instance %s for image %s"%(instance, imagename))
elif len(args) == 1:
containername = args[0]
imagename = extract_image_name(containername)
if imagename:
start_container(imagename, containername)
else:
common.fail("Unknown. Use 'jockler start {new|last|stable} IMAGE' or 'jockler start CONTAINER'")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-i", metavar='in-directory', action="store", help="input directory argument")
parser.add_argument("-csv", help="output csv file argument", action="store")
parser.add_argument("-k", help="list of keywords argument, surround list with quotes", action="store")
parser.add_argument("-type", help="set the name of the text field you're analyzing", action="store")
parser.add_argument("-bin", help="track binary (0/1) occurrence, default is raw frequency", action="store_true")
try:
args = parser.parse_args()
except IOError as msg:
common.fail(parser.error(str(msg)))
if args.i is None:
common.fail("Please specify input directory.")
else:
corpus = args.i
binary = args.bin
keywords = args.k.lower().split("/")
if args.type is None:
text_type = "Words"
else:
text_type = args.type
frequencies = take_frequencies(corpus, keywords, text_type, binary)
with open(args.csv + '.csv', 'w', newline='', encoding='utf-8') as csv_out:
csvwriter = csv.writer(csv_out, delimiter=',')
row = ["filename", "publication date"]
row.extend(keywords)
if not binary:
row.append("total words")
csvwriter.writerow(row)
sorted_freq = sorted(frequencies, key=operator.itemgetter(1))
print("Writing to CSV\n")
for volume in tqdm.tqdm(sorted_freq):
csvwriter.writerow(volume)
def build(args):
if not common.args_check(args, 1):
common.fail("Specify image name to build !")
imagename = args[0]
if not re.match("^" + common.imagenamepat + "$", imagename):
common.fail(
"Image name can only have letters a-z, A-Z and numbers 0-9.")
dockerfile, build_path = dockerfiles_for(imagename, args)
# Add jcl file first - if cannot be generated, prevents build
add_jcl(imagename, dockerfile)
res, sout, serr = do_build(imagename, dockerfile, build_path, args[1:])
if res == 0:
image_id = get_tagged_image_id(imagename)
append_image_id(imagename, image_id)
exit(res)
def main():
if not common.args_check(sys.argv, 2):
printhelp()
common.fail("Fail - not enough arguments")
action = sys.argv[1]
if action == "build":
image.build(sys.argv[2:])
elif action == "start":
container.start(sys.argv[2:])
elif action == "stop":
container.stop(sys.argv[2:])
elif action == "stable":
stability.stable(sys.argv[2:])
elif action == "list":
listing.listing(sys.argv[2:])
elif action == "cleanup":
removal.cleanup(sys.argv[2:])
elif action == "remove":
removal.remove(sys.argv[2:])
elif action == "attach":
attach.attach(sys.argv[2:])
elif action == "volumes":
volumes.volumes(sys.argv[2:])
elif action == "readme":
readme.dump(sys.argv[2:])
else:
printhelp()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-csv", metavar='in-directory', action="store", help="input directory argument")
parser.add_argument("-o", help="output text file argument", action="store")
try:
args = parser.parse_args()
except IOError:
common.fail("IOError")
common.build_out(args.o)
chrome_opts = webdriver.ChromeOptions()
# set default download dir to specified output dir
prefs = {"download.default_directory": args.o}
chrome_opts.add_experimental_option("prefs", prefs)
# init driver
driver = webdriver.Chrome(chrome_options=chrome_opts)
if args.csv is not None:
urls = get_urls(args.csv)
download_files(driver, urls)
else:
common.fail("Please specify csv file path.")
def start_new_container(imagename, doattach=False):
stop_containers(imagename)
containername = generate_container_name(imagename)
options = load_container_options(imagename)
runmode = "-d"
useexec = False
if doattach:
runmode = "-it"
useexec = True
code, sout, serr = run.call(["docker", "run", runmode, "--name=%s"%containername]+options+[imagename], useexec=useexec)
if code > 0 or not found_running_container(containername):
common.fail("Could not create new container for %s, or could not start created container:\n%s"%(imagename, sout))
# Do not do this on initial docker-run - otherwise, if the entrypoint is faulty
# you get continually restarting containers that are hard to stop
code, sout, serr = run.call( ["docker", "update", "--restart=unless-stopped", containername])
store.write_data("last", imagename, containername)
return containername
def determine_data_type(tavg, tmax, tmin, percent, mean, var):
# init return values so IDE doesn't complain
data_type, y_label, title = '', '', ''
arg_sum = int(tavg) + int(tmax) + int(tmin) + int(percent) + int(
mean) + int(var)
if arg_sum > 1 or arg_sum < 1:
common.fail("Please enter exactly one graphing parameter.")
else:
if tavg:
data_type = 'tf-idf avg'
y_label = "Average TF-IDF"
title = "Average TF-IDF Scores Per Period"
elif tmax:
data_type = 'tf-idf max'
y_label = "Maximum TF-IDF"
title = "Maximum TF-IDF Scores Per Period"
elif tmin:
data_type = 'tf-idf min'
y_label = "Minimum TF-IDF"
title = "Minimum TF-IDF Scores Per Period"
elif percent:
data_type = 'word frequency'
y_label = "Word Frequency"
title = "Word Frequency as Percentage of Total Words Per Period"
elif mean:
data_type = 'average frequency'
y_label = "Average Frequency"
title = "Average Word Frequency Per Period"
elif var:
data_type = 'variance'
y_label = "Variance"
title = "Variance of Word Frequency Per Period"
else:
common.fail(
'You shouldn\'t be able to get here, congratulations!!')
return [data_type, y_label, title]
def initFromXmlElement(self, element, storages):
self.storageId = checkElementAttribute(element, "storage_id")
self.filters = []
self.storage = storages[self.storageId]
verticals = element.getElementsByTagName("vertical")
if len(verticals) != 1:
fail("Bad amount of vertical dimensions")
verticalDimensionName = checkElementAttribute(verticals[0], "dimension")
self.verticalDimension = self.storage.getDimensionByName(verticalDimensionName)
horizontals = element.getElementsByTagName("horizontal")
if len(horizontals) != 1:
fail("Bad amount of horizontal dimensions")
horizontalDimensionName = checkElementAttribute(horizontals[0], "dimension")
self.horizontalDimension = self.storage.getDimensionByName(horizontalDimensionName)
filtersElements = element.getElementsByTagName("filters")
if len(filtersElements) != 1:
fail("Bad amount of filters elements")
filters = filtersElements[0].getElementsByTagName("filter")
for fElement in filters:
dimensionName = checkElementAttribute(fElement, "dimension")
curDimension = self.storage.getDimensionByName(dimensionName)
filterType = checkElementAttribute(fElement, "type")
if filterType == "equality":
equalValueElements = fElement.getElementsByTagName("equal")
equals = []
for e in equalValueElements:
equals.append(checkElementAttribute(e, "value"))
self.filters.append(EqualityFilter(equals, curDimension))
elif filterType == "inequality":
values = {}
if fElement.hasAttribute("from"):
values["from"] = fElement.getAttribute("from")
if fElement.hasAttribute("to"):
values["to"] = fElement.getAttribute("to")
self.filters.append(InequalityFilter(curDimension, values))
else:
fail("Bad filter type")
def volumes(args):
if not common.args_check(args, 2):
common.fail(helpstring)
context = args[0]
imagename = args[1]
if context == "image":
imagevolumes = volumes_for(imagename)
print(os.linesep.join(imagevolumes))
elif context == "container":
instance = "last"
if common.args_check(args, 3):
instance = args[2]
list_container_volumes(imagename, instance)
elif context == "backup":
if not common.args_check(args, 3):
common.fail(helpstring)
systemname = args[1]
imagename = args[2]
backup.do_volume_backup(imagename, systemname)
elif context == "restore":
if not common.args_check(args, 4):
common.fail(helpstring)
systemname = args[1]
imagename = args[2]
archivefile = args[3]
backup.do_volume_restore(imagename, systemname, archivefile)
else:
print(helpstring)
common.fail("Unknown context [%s]" % context)
def args_setup(in_dir, keys, out_dir, text_type, threshold):
if in_dir is None:
common.fail("Please specify input (-i) directory.")
if keys is None:
common.fail("Please specify keywords (-k)")
else:
# TODO: make bigrams configurable in future if needed
key_list = common.build_key_list(keys, False)
if out_dir is None:
common.fail("Please specify output (-o) directory.")
else:
common.build_out(out_dir)
common.build_subdirs(out_dir, key_list, False)
if text_type is None:
text_type = 'Filtered Text Stemmed'
else:
text_type = determine_text_type(text_type.lower())
if threshold is None:
common.fail("Please specify TF-IDF threshold argument (-thresh")
else:
thresh = float(threshold)
return [key_list, text_type, thresh]
def do_volume_backup(imagename, systemname, destination="."):
containername = store.read_data("last", imagename)
if not containername:
common.fail("No candidate container for image [%s]" % imagename)
destination = os.path.abspath(destination)
mountpoints = volumes.mountdata_of(containername, "Destination")
archivename = "%s-as-of-%s.tar.gz" % (containername, common.timestring())
if systemname == "linux":
res, sout, serr = do_linux_volume_backup(containername, destination,
archivename, mountpoints)
elif systemname == "windows":
#TODO - need a windows version
common.fail("Not yet implemented")
else:
common.fail("Unkown system type [%s]" % systemname)
if res == 0:
print("Backup created in %s" %
os.path.sep.join([destination, archivename]))
else:
common.fail("Backup operation failed!")
def read_one_sequence(defects, get_line, pushback, fname):
(when, msg) = get_line()
if not START_CRASHING_RE.match(msg):
common.fail("Lost sequencing at %s in %s" % (msg, fname))
start_time = when
expected_start = when
nr_produced = 0
r = []
key = None
have_compile = False
while True:
(when, msg) = get_line()
if msg == "finish crashing thread":
end_time = when
break
if msg == plt:
r.append(plt)
continue
if msg == "done PLT check":
continue
w = msg.split()
if w[0:2] == ["high", "water:"]:
continue
if w[0] == "Marker1":
continue
assert w[0] == "start"
start = when
key = " ".join(w[1:])
if key == "compile crashing machine":
if have_compile:
key = "ssa crashing machine"
have_compile = None
else:
assert have_compile == False
have_compile = True
assert key in bubble_keys
if not defects.has_key(key):
defects[key] = 0
defects[key] += start - expected_start
(when, msg) = get_line()
rr = {"key": key, "duration": when - start}
if msg == "timeout":
rr["failed"] = "timeout"
elif msg == "out of memory":
rr["failed"] = "oom"
else:
w = msg.split()
if w[0] == "produced" and w[2] == "interfering" and w[3] == "CFGs":
assert nr_produced == 0
nr_produced = int(w[1])
(what, msg) = get_line()
rr["duration"] = when - start
#assert msg.split()[0] in ["stop", "finish"] and " ".join(msg.split()[1:]) == key
rr["failed"] = None
expected_start = when
r.append(rr)
l = get_line()
if l != None:
if l[1] in [no_interfering_stores, early_out]:
assert nr_produced == 0
r.append(l[1])
else:
pushback("%f: %s\n" % (l[0], l[1]))
# Check for leftovers
t = get_line()
assert t == None
if not defects.has_key("leftover"):
defects["leftover"] = 0
defects["leftover"] += end_time - expected_start
if key != None:
r.append({"key": key, "duration": end_time - expected_start, "failed": None})
return ((end_time - start_time, r), nr_produced)
def getPlaylist(location_id):
l = Location.from_id(location_id)
try:
return l.playlist().to_json()
except:
return common.fail()
pass
def datatest2():
i = 'potty/'+str(os.urandom(10)).replace('/','|')
def recive(x,y):
tester[x] = y
subscribe('potty/',recive)
x =str(os.urandom(100))
postMessage(i,x)
time.sleep((random.random()/100)+0.05)
if not tester[i] == x:
fail('Message not properly recieved.')
else:
#print("hooray")
pass
for i in range(0,1):
for i in range(0,100):
t = threading.Thread(target=dostuff)
t.start()
t = threading.Thread(target=datatest)
t.start()
t = threading.Thread(target=datatest2)
t.start()
time.sleep(0.05)
if len(messagebus._bus.subscribers) > 5:
fail('It appears that there are old subscriptions in the list for deleted callbacks. Memory Leak!')
suceed("Sucess in testing Message Bus system")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-csv",
help="input csv files argument",
action="store")
parser.add_argument("-txt", help="output text filepath", action="store")
parser.add_argument(
"-y",
help="min/max for year range and increment value, surround with quotes",
action="store")
parser.add_argument(
"-p",
help=
"boolean to analyze by different periods rather than a fixed increment value",
action="store_true")
try:
args = parser.parse_args()
except IOError as msg:
common.fail(msg)
if args.csv is None or len(args.csv.split()) != 2:
common.fail(
"Please enter two csv files after -csv, separated by whitespace.")
else:
csv_files = args.csv.split()
if args.txt is None:
common.fail("Please enter output text file path.")
periods = args.p
range_years = args.y.split()
year_params = common.year_params(range_years, periods)
increment, yrange_min, yrange_max = year_params[0], year_params[
1], year_params[2]
year_list = common.build_year_list(increment, range_years, periods,
yrange_max, yrange_min)
first = build_samples(csv_files[0], year_list, yrange_min, yrange_max)
x1 = first[0]
n1 = first[1]
second = build_samples(csv_files[1], year_list, yrange_min, yrange_max)
x2 = second[0]
n2 = second[1]
diff_props = []
critical = scipy.stats.norm.ppf(1 - (0.05 / 2))
# calculate chi-squared and p values
for year in tqdm.tqdm(year_list):
vals = diff_props_test(x1[year], n1[year], x2[year], n2[year])
z = vals[0]
p_val = vals[1]
significance = scipy.stats.norm.cdf(z)
diff_props.append((z, p_val, significance))
with open(args.txt + '.txt', 'w') as txt_out:
for i in range(len(diff_props) - 1):
txt_out.write("Period: {0} - {1}".format(str(
year_list[i]), str(year_list[i + 1])) + "\n")
txt_out.write("Z-score: " + str(diff_props[i][0]) + "\n")
txt_out.write("P value: " + str(diff_props[i][1]) + "\n")
txt_out.write("Significance: " + str(diff_props[i][2]) + "\n")
txt_out.write("Critical: " + str(critical) + "\n\n")
from common import fail,suceed
import persistancefiles
p = persistancefiles.TestHook()
p.write("some/place","test")
if not p.get("some/place") == "test":
fail('')
p.append("some/place","test2")
if not p.get("some/place",index = 1) == "test2":
fail('')
if not p.ls("some") == ['place']:
fail('Added key to something but it did not show in the directory listing')
p.remove("some")
if not p.whatis("some/place") == None:
fail(p.whatis("some/place"))
print("sucess in testing persistance files")
from src import newevt,messagebus,modules_state
import time
modules_state.scopes['x'] = {}
#Create an event that sets y to 0 if it is 1
x = newevt.Event("y==1","global y\ny=0",locals(),setup="y=0")
#Register event with polling.
x.register()
#Set y to 1
x.pymodule.y = 1
time.sleep(0.1)
#y should immediately be set back to 0 at the next polling cycle
if x.pymodule.y == 1:
common.fail("Edge-Triggered Event did nothing")
#Unregister event and make sure it really goes away
x.unregister()
x.pymodule.y =1
if not x.pymodule.y == 1:
common.fail("Edge-Triggered Event did not go away when unregistered")
##Testing one time events
x.pymodule.y = False
def f():
return x.pymodule.y
def g():
x.pymodule.y = False
print('anus')
import common
import src.unitsofmeasure as unitsofmeasure
import random
x = unitsofmeasure.timeIntervalFromString("1 minute")
if not x == 60:
common.fail(x)
x = unitsofmeasure.timeIntervalFromString("1 minute 10 seconds")
if not x == 70:
common.fail(x)
x = unitsofmeasure.timeIntervalFromString("1 minute and 10 secondsbutwithanextra42milliseconds")
if not x == 70.042:
common.fail(x)
if not unitsofmeasure.formatTimeInterval(60.1,2) == '1 minute, 100 milliseconds':
common.fail()
if not unitsofmeasure.strToIntWithSIMultipliers('10k') ==10000:
common.fail()
x = unitsofmeasure.DayOfWeek("Tuesday")
if not x == 1:
common.fail()
if not x == 'tue':
common.fail()
if not x == 'Tuesday':
common.fail()
cherrypy.HTTPRedirect = BSException
from modules import ActiveModules
wb = modules.WebInterface()
######################################################################################
#Add a new module
try:
wb.newmoduletarget(name='TEST')
except BSException:
pass
#Make sure the module shows up in the index
if not 'TEST' in wb.index():
common.fail("Added new module, but it did not show up in the index")
######################################################################################
#Create a page called testpage
try:
wb.module('TEST','addresourcetarget','page',name="testpage")
except BSException:
pass
#View the module's main page and make sure our new page shows up.
if not 'testpage' in wb.module('TEST'):
common.fail("Made a new module but it didn't show up in the module's index")
######################################################################################
#Create a new event
try:
wb.module('TEST','addresourcetarget','event',name="testevent")
except BSException:
import newevt
import time
import messagebus
#Create an event that sets y to 0 if it is 1
x = newevt.Event("y==1","y=0",locals())
#Register event with polling.
x.register()
#Set y to 1
y = 1
time.sleep(0.05)
#y should immediately be set back to 0 at the next polling cycle
if y == 1:
common.fail("Edge-Triggered Event did nothing")
#Unregister event and make sure it really goes away
x.unregister()
y =1
if not y == 1:
common.fail("Edge-Triggered Event did not go away when unregistered")
#Same exact thing exept we use the onchange
x = newevt.Event("!onchange y","y=5",locals())
#Register event with polling.
