def __init__(self, dimension, kwargs):
   self.values = kwargs
   if not self.values.has_key('from') and not self.values.has_key('to'):
     fail("Bad inequality filter")
   self.dimension = dimension
   self.dimensionTable = dimension.table
   self.dimensionField = dimension.field
Example #2
0
    def writeData(self, events_data):

        self.report()

        signals = {}
        # Add all the signals
        for signal in self.signal_elements:
            try:
                signal_value = getattr(self, signal)
                if signal_value is not None:
                    signals[signal] = signal_value
            except:
                fail("Sensor: '" + self.name + "' does not contain member: " +
                     signal)

        # For all events linked with this signal, run isTriggered() and add to response
        events_triggered = {}
        for event in events_data:
            if self.name in event.attached_sensors:
                try:
                    if event.isTriggered():
                        events_triggered[event.PARAM__name] = event.getDict()
                except Exception as e:
                    print("Error while running isTriggered() for event: " +
                          str(event.PARAM__info) + "\n" + str(e))

        response = {}
        if len(signals) > 0:
            response['signals'] = signals
        if len(events_triggered) > 0:
            response['events'] = events_triggered

        self.afterReport()
        return response
Example #3
0
def do_login_admin(browser):
    common.info("Processing login..")
    function_result = True
    # go to the admin login page and wait for page to be loaded
    browser.get("http://localhost/litecart/admin/")
    WebDriverWait(browser, 10).until(
        EC.presence_of_element_located(
            (By.CSS_SELECTOR, "button[value=Login]")))
    # enter credentials
    login_field = browser.find_element_by_name("username")
    login_field.clear()
    login_field.send_keys("admin")
    password_field = browser.find_element_by_name("password")
    password_field.clear()
    password_field.send_keys("admin")
    # press [Login] button and wait for next page to be loaded
    browser.find_element_by_css_selector('button[name="login"]').click()
    WebDriverWait(browser, 10).until(
        EC.element_to_be_clickable((By.CSS_SELECTOR, "li#widget-discussions")))
    if (browser.find_element_by_css_selector("li#widget-discussions")):
        function_result = True
        common.success("Login successful")
    else:
        function_result = False
        common.fail("Cannot login to admin page")
    return function_result
Example #4
0
def build_year_list(in_dir):
    year_lists = []
    for subdir, dirs, files in os.walk(in_dir):
        for file in files:
            if file[0] != '.' and file[-5:] == '.json':
                with open(in_dir + "/" + file, 'r',
                          encoding='utf8') as in_file:
                    jsondata = json.load(in_file)
                    year_lists.append(sorted(jsondata['year list']))
            elif file[0] != '.' and file[-4] == '.csv':
                with open(in_dir + "/" + file, 'r',
                          encoding='utf8') as in_file:
                    read_csv = csv.reader(in_file, delimiter=',')
                    for row in read_csv:
                        if row[0] == "word" and row[1] == "tf-idf avg":
                            years = row[7].split()
                            year_list = sorted([int(year) for year in years])
                            year_lists.append(year_list)
    for year_list in year_lists:
        if year_list != year_lists[0]:
            common.fail(
                "One of your files has a different year list from the others."
                +
                " Please make sure all your files contain the same year lists."
            )
    # just return first list in year_lists if they're all the same
    return year_lists[0]
Example #5
0
def read_one_sequence(defects, get_line, _pushback, fname):
    t = get_line()
    if t == None:
        return None
    (when, msg) = t
    if not START_INTERFERING_RE.match(msg):
        common.fail("Lost sequencing at %s in %s" % (msg, fname))
    start_time = when
    expected_start = when
    r = []
    key = None
    while True:
        (when, msg) = get_line()
        if msg == "stop interfering CFG":
            end_time = when
            (_, msg) = get_line()
            w = msg.split()
            if w[:2] != ["high", "water:"]:
                common.fail("Lost sequencing 2 at %s" % msg)
            break
        w = msg.split()
        if msg in [early_out, satisfiable, unsatisfiable, ic_atomic_false]:
            r.append(msg)
            continue
        assert w[0] == "start"
        start = when
        key = " ".join(w[1:])
        assert key in bubble_keys

        if not defects.has_key(key):
            defects[key] = 0
        defects[key] += start - expected_start

        t = get_line()
        if t == None:
            return None
        (when, msg) = t
        rr = {"key": key, "duration": when - start}
        if msg == "timeout":
            rr["failed"] = "timeout"
        elif msg == "out of memory":
            rr["failed"] = "oom"
        else:
            assert msg.split()[0] == "stop" and " ".join(msg.split()[1:]) == key
            rr["failed"] = None
        expected_start = when
        r.append(rr)
        if rr["failed"]:
            end_time = when
            break
    # Check for leftovers
    t = get_line()
    assert t == None

    if not defects.has_key("leftover"):
        defects["leftover"] = 0
    defects["leftover"] += end_time - expected_start
    if key != None:
        r.append({"key": key, "duration": end_time - expected_start, "failed": None})
    return (end_time - start_time, r)
Example #6
0
def remove(args, keep_images=[], keep_containers=[]):
    if not common.args_check(args, 1):
        common.fail(
            "Remove all containers and images associated with an image.\n\nUsage:\n\n    jockler remove IMAGENAME"
        )

    imagename = args[0]

    allimages = listing.get_image_list_for(imagename)
    allcontainers = listing.get_container_list_for(imagename)

    for k in set(keep_containers):
        try_remove(k, allcontainers)
    for k in set(keep_images):
        try_remove(k, allimages)

    if not confirm_removal(allcontainers, allimages, keep_containers,
                           keep_images):
        common.fail("ABORTED")

    # Remove the image tag
    run.call(["docker", "rmi", imagename])

    run.call(["docker", "rm"] + allcontainers, stdout=sys.stdout, silent=True)

    run.call(["docker", "image", "rm"] + allimages,
             stdout=sys.stdout,
             silent=True)
    store.cleanup_images(imagename)
Example #7
0
def stop_containers(imagename):
    containernames = get_running_containers(imagename)

    if len(containernames) > 0:
        code, sout, serr = run.call( ["docker", "stop"] + containernames )
        if code > 0:
            common.fail("Error stopping container(s) !\n%s"%(sout))

        code, sout, serr = run.call( ["docker", "update", "--restart=no"] + containernames )
Example #8
0
def list_container_volumes(imagename, instance):
    if not instance in ["stable", "last"]:
        common.fail("Invalid instance [%s] - use last or stable" % instance)

    containername = store.read_data(instance, imagename)
    if not containername:
        common.fail("No candidate [%s] for image [%s]" % (instance, imagename))

    print(os.linesep.join(mountdata_of(containername, "Name")))
Example #9
0
def get_image_of(containername):
    res,sout,serr = run.call(["docker", "ps", "-a", "--format", "{{.Image}}"] + ps_filter(containername), silent=True)
    images = sout.strip().split("\n")

    common.remove_empty_strings(images)

    if len(images) != 1:
        common.fail("Could not find single image for %s"%containername)

    return images[0]
Example #10
0
def listing(args):
    if not common.args_check(args, 2):
        common.fail(helpstr)

    category = args[0]
    imagename = common.item(args, 1)

    if imagename == ".all" or imagename == None:
        list_all(category)
    else:
        list_on_image(imagename, category)
Example #11
0
def stable(args):

    if not common.args_check(args, 1):
        common.fail(helpstr)
        
    imagename = args[0]

    if common.args_check(args, 2) and imagename == ":show":
        imagename = args[1]
        print_stable_container(imagename)
    else:
        mark_stable_container(imagename)
Example #12
0
def mark_stable_container(imagename):

    containernames = container.get_running_containers(imagename)

    if len(containernames) != 1:
        print(containernames)
        common.fail("Could not identify single container to mark as stable")

    containername = containernames[0]

    store.write_data("stable", imagename, containername)
    print("Marked %s as stable. Start it again using 'jockler start stable %s'"%(containername, imagename))
Example #13
0
def read_one_sequence(defects):
    try:
        (when, msg) = common.get_line()
    except StopIteration:
        return None
    w = msg.split()
    if w[:-1] != ["start", "build", "enforcer"]:
        common.fail("Lost sequencing at %s" % msg)
    start_time = when
    expected_start = when
    nr_produced = 0
    r = []
    key = None
    while True:
        (when, msg) = common.get_line()
        if msg == "stop build enforcer":
            end_time = when
            if key != None:
                r.append({"key": key, "duration": end_time - expected_start, "failed": None})
            break
        w = msg.split()
        assert w[0] == "start"
        start = when
        key = " ".join(w[1:])
        assert key in bubble_keys

        if not defects.has_key(key):
            defects[key] = 0
        assert start >= expected_start
        defects[key] += start - expected_start

        (when, msg) = common.get_line()
        assert when > start
        rr = {"key": key, "duration": when - start}
        r.append(rr)
        if msg == "timeout":
            rr["failed"] = "timeout"
            end_time = when
            break
        elif msg == "out of memory":
            rr["failed"] = "oom"
            end_time = when
            break
        else:
            w = msg.split()
            assert msg.split()[0] == "stop" and " ".join(msg.split()[1:]) == key
            rr["failed"] = None
        expected_start = when
    if not defects.has_key("leftover"):
        defects["leftover"] = 0
    assert end_time > expected_start
    defects["leftover"] += end_time - expected_start
    return ((end_time - start_time, r), nr_produced)
def parseConfigFile(filename="./config.xml"):
  dom = parse(filename)
  rootElement = dom.documentElement
  if rootElement.tagName != "storages":
    fail("Bad xml!")

  storageElements = rootElement.getElementsByTagName("storage")
  storages = {}
  for element in storageElements:
    storage = Storage(element)
    storages[storage.getId()] = storage

  return storages
 def datatest():
     i = str(os.urandom(10)).replace('/','|')
     def recive(x,y):
         tester[x] = y
     
     subscribe(i,recive)
     x =str(os.urandom(100))
     postMessage(i,x)
     time.sleep((random.random()/100)+0.05)
     if not tester[i] == x:
         fail('Message not properly recieved')
     else:
         pass
  def getTable(self, dimensions, filters = []):
    if len(dimensions) != 2:
      fail("Bad dimensions size")

    horizontalFilters = []
    verticalFilters = []
    for f in filters:
      if f.dimension.name == self.dimensions[dimensions[0]].name:
        horizontalFilters.append(f)
      if f.dimension.name == self.dimensions[dimensions[1]].name:
        verticalFilters.append(f)
    horizontalNames = self.getNamesForOneDimension(self.dimensions[dimensions[0]], horizontalFilters)
    verticalNames = self.getNamesForOneDimension(self.dimensions[dimensions[1]], verticalFilters)

    connection = sqlite3.connect(self.path)
    query = "select "
    for dimensionName in dimensions:
      dimension = self.dimensions[dimensionName]
      query += dimension.table + "." + dimension.field + ", "
    query += "sum(" + self.aggregatedValueField + ") "
    query += " from " + self.factTable
    query += self.joins

    query += " where 1=1 "
    for f in filters:
      query += f.generateQueryString()

    query += " group by "
    for dimensionName in dimensions:
      dimension = self.dimensions[dimensionName]
      query += dimension.table + "." + dimension.field + ", "
    query = query[:-2]

    cursor = connection.execute(query)
    resultDict = {}
    for row in cursor:
      d = resultDict.get(row[0], {})
      d[row[1]] = row[2]
      resultDict[row[0]] = d

    ans = []
    for h in horizontalNames:
      last = []
      for v in verticalNames:
        if resultDict.has_key(h) and resultDict[h].has_key(v):
          last.append(resultDict[h][v])
        else:
          last.append(0)
      ans.append(last)

    return horizontalNames, verticalNames, ans
Example #17
0
def stop(args):
    if not common.args_check(args, 1):
        common.fail("Unknown sequence for stop: %s" % ' '.join(args))

    imagename = args[0]
    forcemode = False
    if imagename == "-f" and common.item(args, 1, None) != None:
        forcemode = True

    if forcemode:
        for containername in args[1:]:
            force_stop(containername)
    else:
        stop_containers(imagename)
Example #18
0
def do_volume_restore(imagename, systemname, archivefile, destination="."):
    destination = os.path.abspath(destination)
    archive_fullpath = os.path.sep.join([destination, archivefile])

    volumes_array = options.extract_section(imagename, "-v", "volumes")
    volume_mountpoints = options.extract_container_side(imagename, "volumes")

    if not os.path.isfile(archive_fullpath):
        common.fail("Could not find [%s]" % archive_fullpath)

    if systemname == "linux":
        res, sout, serr = do_linux_clear_data(volumes_array,
                                              volume_mountpoints)
        if res >= 0:
            res, sout, serr = do_linux_volume_restore(volumes_array,
                                                      destination, archivefile)

    elif systemname == "windows":
        #TODO - need a windows version
        common.fail("Not yet implemented")
    else:
        common.fail("Unkown system type [%s]" % systemname)

    if res == 0:
        print("Restore complete from %s" % archivefile)
    else:
        common.fail("Restore operation failed!")
Example #19
0
    def datatest():
        i = str(os.urandom(10)).replace('/', '|')

        def recive(x, y):
            tester[x] = y

        subscribe(i, recive)
        x = str(os.urandom(100))
        postMessage(i, x)
        time.sleep((random.random() / 100) + 0.05)
        if not tester[i] == x:
            fail('Message not properly recieved')
        else:
            pass
 def datatest2():
     i = 'potty/'+str(os.urandom(10)).replace('/','|')
     def recive(x,y):
         tester[x] = y
     
     subscribe('potty/',recive)
     x =str(os.urandom(100))
     postMessage(i,x)
     time.sleep((random.random()/100)+0.05)
     if not tester[i] == x:
         fail('Message not properly recieved.')
     else:
         #print("hooray")
         pass
Example #21
0
    def datatest2():
        i = 'potty/' + str(os.urandom(10)).replace('/', '|')

        def recive(x, y):
            tester[x] = y

        subscribe('potty/', recive)
        x = str(os.urandom(100))
        postMessage(i, x)
        time.sleep((random.random() / 100) + 0.05)
        if not tester[i] == x:
            fail('Message not properly recieved.')
        else:
            #print("hooray")
            pass
Example #22
0
def select_from_dropdown_list(browser, data_to_set):
    value_to_set = data_to_set["value"]
    # if no value should be set then just return
    if (not value_to_set):
        return True
    field_to_be_set = browser.find_element_by_css_selector(data_to_set["location"])
    select_field = Select(field_to_be_set)
    try:
        WebDriverWait(browser, 15).until(EC.element_to_be_clickable((By.CSS_SELECTOR, data_to_set["location"])))
        select_field.select_by_value(value_to_set)
        if (field_to_be_set.get_property("value") == value_to_set):
            common.info("Set %s = '%s': ok" % (data_to_set["location"], data_to_set["description"]))
        return True
    except:
        common.fail("Set %s = '%s'" % (data_to_set["location"], data_to_set["description"]))
        return False
Example #23
0
def determine_index(data_type):
    index = 0
    if data_type == 'tf-idf avg':
        index = 1
    elif data_type == 'tf-idf max':
        index = 2
    elif data_type == 'tf-idf min':
        index = 3
    elif data_type == 'word frequency':
        index = 4
    elif data_type == 'average frequency':
        index = 5
    elif data_type == 'variance':
        index = 6
    else:
        common.fail('You shouldn\'t be able to get here, congratulations!!')
    return index
Example #24
0
def start_container(imagename, containername, doattach=False):
    stop_containers(imagename)
    store.write_data("last", imagename, containername)

    runmode = []
    useexec = False
    if doattach:
        runmode.append("-i")
        useexec = True

    print("Starting %s"%containername)

    code, sout, serr = run.call( ["docker", "start", containername]+runmode, stdout=sys.stdout, stderr=sys.stderr, useexec=useexec )

    if code > 0 or not found_running_container(containername):
        common.fail("Could not start container %s - try starting with 'attach' mode'\n%s"%(containername,sout))

    code, sout, serr = run.call( ["docker", "update", "--restart=unless-stopped", containername])
Example #25
0
def cleanup(args):
    if not common.args_check(args, 1):
        common.fail(
            "Remove all old containers and images, except for last and stable.\n\nUsage:\n\n    jockler cleanup IMAGENAME"
        )

    imagename = args[0]

    keep_images = []
    keep_containers = []

    for instance in ["stable", "last"]:
        container = store.read_data(instance, imagename)
        if container:
            keep_images.append(imagename)
            keep_containers.append(container)

    remove([imagename], keep_images, keep_containers)
Example #26
0
def attach(args):
    if not common.args_check(args, 1):
        common.fail("Usage:\n  jockler attach IMAGENAME [COMMAND]")

    imagename = args[0]

    shellname = "bash"
    if common.args_check(args, 2):
        shellname = args[1]

    containernames = container.get_running_containers(imagename)

    if len(containernames) != 1:
        common.fail("Could not identify single container to attach to:\n%s" %
                    (str(containernames)))

    command_tokens = ["docker", "exec", "-it", containernames[0], shellname]
    os.execvp(command_tokens[0], command_tokens)
  def __init__(self, element):
    self.id = checkElementAttribute(element, "id")
    self.path = checkElementAttribute(element, "path")
    self.aggregatedValueName = checkElementAttribute(element, "aggregated_value_name")
    self.factTable = checkElementAttribute(element, "fact_table")
    self.aggregatedValueField = checkElementAttribute(element, "aggregated_value_field")
    self.dimensions = {}
    self.joins = ""

    dimensionsHolder = element.getElementsByTagName("dimensions")
    if len(dimensionsHolder) > 1:
      fail("Too many dimensions elements!")
    if len(dimensionsHolder) == 0 :
      fail("No dimensions elements!")

    dimensionsElements = dimensionsHolder[0].getElementsByTagName("dimension")
    for element in dimensionsElements:
      dimension = Dimension(element)
      self.joins += " join " + dimension.table + " on " + dimension.factFk + "=" + dimension.table + ".id"
      self.dimensions[dimension.name] = dimension
Example #28
0
def start(args):
    if len(args) >= 2:
        instance = args[0]
        imagename = args[1]

        doattach = common.item(args, 2, False)
        if doattach == "attach":
            doattach=True

        if not instance in ["new", "last", "stable"]:
            common.fail("Incorrect instance name. Please use 'new', 'last', or 'stable' ")

        if instance == "new":
            containername = start_new_container(imagename, doattach)
        else:
            containername = store.read_data(instance, imagename)
            if containername:
                start_container(imagename, containername, doattach)
            else:
                common.fail("No instance %s for image %s"%(instance, imagename))

    elif len(args) == 1:
        containername = args[0]
        imagename = extract_image_name(containername)
        if imagename:
            start_container(imagename, containername)
    
    else:
        common.fail("Unknown. Use 'jockler start {new|last|stable} IMAGE' or 'jockler start CONTAINER'")
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-i", metavar='in-directory', action="store", help="input directory argument")
    parser.add_argument("-csv", help="output csv file argument", action="store")
    parser.add_argument("-k", help="list of keywords argument, surround list with quotes", action="store")
    parser.add_argument("-type", help="set the name of the text field you're analyzing", action="store")
    parser.add_argument("-bin", help="track binary (0/1) occurrence, default is raw frequency", action="store_true")

    try:
        args = parser.parse_args()
    except IOError as msg:
        common.fail(parser.error(str(msg)))

    if args.i is None:
        common.fail("Please specify input directory.")
    else:
        corpus = args.i

    binary = args.bin
    keywords = args.k.lower().split("/")
    
    if args.type is None:
        text_type = "Words"
    else:
        text_type = args.type

    frequencies = take_frequencies(corpus, keywords, text_type, binary)

    with open(args.csv + '.csv', 'w', newline='', encoding='utf-8') as csv_out:
        csvwriter = csv.writer(csv_out, delimiter=',')
        row = ["filename", "publication date"]
        row.extend(keywords)
        if not binary:
            row.append("total words")
        csvwriter.writerow(row)
        sorted_freq = sorted(frequencies, key=operator.itemgetter(1))
        print("Writing to CSV\n")
        for volume in tqdm.tqdm(sorted_freq):
            csvwriter.writerow(volume)
Example #30
0
def build(args):
    if not common.args_check(args, 1):
        common.fail("Specify image name to build !")

    imagename = args[0]

    if not re.match("^" + common.imagenamepat + "$", imagename):
        common.fail(
            "Image name can only have letters a-z, A-Z and numbers 0-9.")

    dockerfile, build_path = dockerfiles_for(imagename, args)

    # Add jcl file first - if cannot be generated, prevents build
    add_jcl(imagename, dockerfile)

    res, sout, serr = do_build(imagename, dockerfile, build_path, args[1:])

    if res == 0:
        image_id = get_tagged_image_id(imagename)
        append_image_id(imagename, image_id)

    exit(res)
Example #31
0
def main():
    if not common.args_check(sys.argv, 2):
        printhelp()
        common.fail("Fail - not enough arguments")
    action = sys.argv[1]

    if action == "build":
        image.build(sys.argv[2:])

    elif action == "start":
        container.start(sys.argv[2:])

    elif action == "stop":
        container.stop(sys.argv[2:])

    elif action == "stable":
        stability.stable(sys.argv[2:])

    elif action == "list":
        listing.listing(sys.argv[2:])

    elif action == "cleanup":
        removal.cleanup(sys.argv[2:])

    elif action == "remove":
        removal.remove(sys.argv[2:])

    elif action == "attach":
        attach.attach(sys.argv[2:])

    elif action == "volumes":
        volumes.volumes(sys.argv[2:])

    elif action == "readme":
        readme.dump(sys.argv[2:])

    else:
        printhelp()
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-csv", metavar='in-directory', action="store", help="input directory argument")
    parser.add_argument("-o", help="output text file argument", action="store")

    try:
        args = parser.parse_args()
    except IOError:
        common.fail("IOError")

    common.build_out(args.o)
    chrome_opts = webdriver.ChromeOptions()
    # set default download dir to specified output dir
    prefs = {"download.default_directory": args.o}
    chrome_opts.add_experimental_option("prefs", prefs)
    # init driver
    driver = webdriver.Chrome(chrome_options=chrome_opts)

    if args.csv is not None:
        urls = get_urls(args.csv)
        download_files(driver, urls)
    else:
        common.fail("Please specify csv file path.")
Example #33
0
def start_new_container(imagename, doattach=False):
    stop_containers(imagename)
    containername = generate_container_name(imagename)
    options = load_container_options(imagename)

    runmode = "-d"
    useexec = False
    if doattach:
        runmode = "-it"
        useexec = True

    code, sout, serr = run.call(["docker", "run", runmode, "--name=%s"%containername]+options+[imagename], useexec=useexec)

    if code > 0 or not found_running_container(containername):
        common.fail("Could not create new container for %s, or could not start created container:\n%s"%(imagename, sout))

    # Do not do this on initial docker-run - otherwise, if the entrypoint is faulty
    #   you get continually restarting containers that are hard to stop
    code, sout, serr = run.call( ["docker", "update", "--restart=unless-stopped", containername])

    store.write_data("last", imagename, containername)

    return containername
Example #34
0
def determine_data_type(tavg, tmax, tmin, percent, mean, var):
    # init return values so IDE doesn't complain
    data_type, y_label, title = '', '', ''
    arg_sum = int(tavg) + int(tmax) + int(tmin) + int(percent) + int(
        mean) + int(var)
    if arg_sum > 1 or arg_sum < 1:
        common.fail("Please enter exactly one graphing parameter.")
    else:
        if tavg:
            data_type = 'tf-idf avg'
            y_label = "Average TF-IDF"
            title = "Average TF-IDF Scores Per Period"
        elif tmax:
            data_type = 'tf-idf max'
            y_label = "Maximum TF-IDF"
            title = "Maximum TF-IDF Scores Per Period"
        elif tmin:
            data_type = 'tf-idf min'
            y_label = "Minimum TF-IDF"
            title = "Minimum TF-IDF Scores Per Period"
        elif percent:
            data_type = 'word frequency'
            y_label = "Word Frequency"
            title = "Word Frequency as Percentage of Total Words Per Period"
        elif mean:
            data_type = 'average frequency'
            y_label = "Average Frequency"
            title = "Average Word Frequency Per Period"
        elif var:
            data_type = 'variance'
            y_label = "Variance"
            title = "Variance of Word Frequency Per Period"
        else:
            common.fail(
                'You shouldn\'t be able to get here, congratulations!!')
    return [data_type, y_label, title]
  def initFromXmlElement(self, element, storages):
    self.storageId = checkElementAttribute(element, "storage_id")
    self.filters = []
    self.storage = storages[self.storageId]

    verticals = element.getElementsByTagName("vertical")
    if len(verticals) != 1:
      fail("Bad amount of vertical dimensions")
    verticalDimensionName = checkElementAttribute(verticals[0], "dimension")
    self.verticalDimension = self.storage.getDimensionByName(verticalDimensionName)

    horizontals = element.getElementsByTagName("horizontal")
    if len(horizontals) != 1:
      fail("Bad amount of horizontal dimensions")
    horizontalDimensionName = checkElementAttribute(horizontals[0], "dimension")
    self.horizontalDimension = self.storage.getDimensionByName(horizontalDimensionName)

    filtersElements = element.getElementsByTagName("filters")
    if len(filtersElements) != 1:
      fail("Bad amount of filters elements")
    filters = filtersElements[0].getElementsByTagName("filter")

    for fElement in filters:
      dimensionName = checkElementAttribute(fElement, "dimension")
      curDimension = self.storage.getDimensionByName(dimensionName)
      filterType = checkElementAttribute(fElement, "type")
      if filterType == "equality":
        equalValueElements = fElement.getElementsByTagName("equal")
        equals = []
        for e in equalValueElements:
          equals.append(checkElementAttribute(e, "value"))
        self.filters.append(EqualityFilter(equals, curDimension))
      elif filterType == "inequality":
        values = {}
        if fElement.hasAttribute("from"):
          values["from"] = fElement.getAttribute("from")
        if fElement.hasAttribute("to"):
          values["to"] = fElement.getAttribute("to")
        self.filters.append(InequalityFilter(curDimension, values))
      else:
        fail("Bad filter type")
Example #36
0
def volumes(args):
    if not common.args_check(args, 2):
        common.fail(helpstring)

    context = args[0]
    imagename = args[1]

    if context == "image":
        imagevolumes = volumes_for(imagename)
        print(os.linesep.join(imagevolumes))

    elif context == "container":
        instance = "last"
        if common.args_check(args, 3):
            instance = args[2]

        list_container_volumes(imagename, instance)

    elif context == "backup":
        if not common.args_check(args, 3):
            common.fail(helpstring)

        systemname = args[1]
        imagename = args[2]

        backup.do_volume_backup(imagename, systemname)

    elif context == "restore":
        if not common.args_check(args, 4):
            common.fail(helpstring)

        systemname = args[1]
        imagename = args[2]
        archivefile = args[3]

        backup.do_volume_restore(imagename, systemname, archivefile)

    else:
        print(helpstring)
        common.fail("Unknown context [%s]" % context)
Example #37
0
def args_setup(in_dir, keys, out_dir, text_type, threshold):
    if in_dir is None:
        common.fail("Please specify input (-i) directory.")
    if keys is None:
        common.fail("Please specify keywords (-k)")
    else:
        # TODO: make bigrams configurable in future if needed
        key_list = common.build_key_list(keys, False)
    if out_dir is None:
        common.fail("Please specify output (-o) directory.")
    else:
        common.build_out(out_dir)
        common.build_subdirs(out_dir, key_list, False)
    if text_type is None:
        text_type = 'Filtered Text Stemmed'
    else:
        text_type = determine_text_type(text_type.lower())
    if threshold is None:
        common.fail("Please specify TF-IDF threshold argument (-thresh")
    else:
        thresh = float(threshold)
    return [key_list, text_type, thresh]
Example #38
0
def do_volume_backup(imagename, systemname, destination="."):
    containername = store.read_data("last", imagename)
    if not containername:
        common.fail("No candidate container for image [%s]" % imagename)

    destination = os.path.abspath(destination)
    mountpoints = volumes.mountdata_of(containername, "Destination")

    archivename = "%s-as-of-%s.tar.gz" % (containername, common.timestring())

    if systemname == "linux":
        res, sout, serr = do_linux_volume_backup(containername, destination,
                                                 archivename, mountpoints)
    elif systemname == "windows":
        #TODO - need a windows version
        common.fail("Not yet implemented")
    else:
        common.fail("Unkown system type [%s]" % systemname)

    if res == 0:
        print("Backup created in %s" %
              os.path.sep.join([destination, archivename]))
    else:
        common.fail("Backup operation failed!")
Example #39
0
def read_one_sequence(defects, get_line, pushback, fname):
    (when, msg) = get_line()
    if not START_CRASHING_RE.match(msg):
        common.fail("Lost sequencing at %s in %s" % (msg, fname))
    start_time = when
    expected_start = when
    nr_produced = 0
    r = []
    key = None
    have_compile = False
    while True:
        (when, msg) = get_line()
        if msg == "finish crashing thread":
            end_time = when
            break
        if msg == plt:
            r.append(plt)
            continue
        if msg == "done PLT check":
            continue
        w = msg.split()
        if w[0:2] == ["high", "water:"]:
            continue
        if w[0] == "Marker1":
            continue
        assert w[0] == "start"
        start = when
        key = " ".join(w[1:])
        if key == "compile crashing machine":
            if have_compile:
                key = "ssa crashing machine"
                have_compile = None
            else:
                assert have_compile == False
                have_compile = True
        assert key in bubble_keys

        if not defects.has_key(key):
            defects[key] = 0
        defects[key] += start - expected_start

        (when, msg) = get_line()
        rr = {"key": key, "duration": when - start}
        if msg == "timeout":
            rr["failed"] = "timeout"
        elif msg == "out of memory":
            rr["failed"] = "oom"
        else:
            w = msg.split()
            if w[0] == "produced" and w[2] == "interfering" and w[3] == "CFGs":
                assert nr_produced == 0
                nr_produced = int(w[1])
                (what, msg) = get_line()
                rr["duration"] = when - start
            #assert msg.split()[0] in ["stop", "finish"] and " ".join(msg.split()[1:]) == key
            rr["failed"] = None
        expected_start = when
        r.append(rr)
        l = get_line()
        if l != None:
            if l[1] in [no_interfering_stores, early_out]:
                assert nr_produced == 0
                r.append(l[1])
            else:
                pushback("%f: %s\n" % (l[0], l[1]))
    # Check for leftovers
    t = get_line()
    assert t == None
    if not defects.has_key("leftover"):
        defects["leftover"] = 0
    defects["leftover"] += end_time - expected_start
    if key != None:
        r.append({"key": key, "duration": end_time - expected_start, "failed": None})
    return ((end_time - start_time, r), nr_produced)
Example #40
0
def getPlaylist(location_id):
    l = Location.from_id(location_id)
    try:
        return l.playlist().to_json()
    except:
        return common.fail()
            pass
            
    def datatest2():
        i = 'potty/'+str(os.urandom(10)).replace('/','|')
        def recive(x,y):
            tester[x] = y
        
        subscribe('potty/',recive)
        x =str(os.urandom(100))
        postMessage(i,x)
        time.sleep((random.random()/100)+0.05)
        if not tester[i] == x:
            fail('Message not properly recieved.')
        else:
            #print("hooray")
            pass
    
    for i in range(0,1):
        for i in range(0,100):
            t = threading.Thread(target=dostuff)
            t.start()
            t = threading.Thread(target=datatest)
            t.start()
            t = threading.Thread(target=datatest2)
            t.start()
            time.sleep(0.05)    
   
if len(messagebus._bus.subscribers) > 5:
    fail('It appears that there are old subscriptions in the list for deleted callbacks. Memory Leak!')
    
suceed("Sucess in testing Message Bus system")
Example #42
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-csv",
                        help="input csv files argument",
                        action="store")
    parser.add_argument("-txt", help="output text filepath", action="store")
    parser.add_argument(
        "-y",
        help="min/max for year range and increment value, surround with quotes",
        action="store")
    parser.add_argument(
        "-p",
        help=
        "boolean to analyze by different periods rather than a fixed increment value",
        action="store_true")

    try:
        args = parser.parse_args()
    except IOError as msg:
        common.fail(msg)

    if args.csv is None or len(args.csv.split()) != 2:
        common.fail(
            "Please enter two csv files after -csv, separated by whitespace.")
    else:
        csv_files = args.csv.split()

    if args.txt is None:
        common.fail("Please enter output text file path.")

    periods = args.p

    range_years = args.y.split()
    year_params = common.year_params(range_years, periods)
    increment, yrange_min, yrange_max = year_params[0], year_params[
        1], year_params[2]

    year_list = common.build_year_list(increment, range_years, periods,
                                       yrange_max, yrange_min)

    first = build_samples(csv_files[0], year_list, yrange_min, yrange_max)
    x1 = first[0]
    n1 = first[1]
    second = build_samples(csv_files[1], year_list, yrange_min, yrange_max)
    x2 = second[0]
    n2 = second[1]

    diff_props = []
    critical = scipy.stats.norm.ppf(1 - (0.05 / 2))

    # calculate chi-squared and p values
    for year in tqdm.tqdm(year_list):
        vals = diff_props_test(x1[year], n1[year], x2[year], n2[year])
        z = vals[0]
        p_val = vals[1]
        significance = scipy.stats.norm.cdf(z)
        diff_props.append((z, p_val, significance))

    with open(args.txt + '.txt', 'w') as txt_out:
        for i in range(len(diff_props) - 1):
            txt_out.write("Period: {0} - {1}".format(str(
                year_list[i]), str(year_list[i + 1])) + "\n")
            txt_out.write("Z-score: " + str(diff_props[i][0]) + "\n")
            txt_out.write("P value: " + str(diff_props[i][1]) + "\n")
            txt_out.write("Significance: " + str(diff_props[i][2]) + "\n")
            txt_out.write("Critical: " + str(critical) + "\n\n")
from common import fail,suceed

import persistancefiles


p = persistancefiles.TestHook()

p.write("some/place","test")

if not p.get("some/place") == "test":
    fail('')

p.append("some/place","test2")

if not p.get("some/place",index = 1) == "test2":
    fail('')

if not p.ls("some") == ['place']:
    fail('Added key to something but it did not show in the directory listing')
    
p.remove("some")

if not p.whatis("some/place") == None:
    fail(p.whatis("some/place"))


print("sucess in testing persistance files")
Example #44
0
from src import newevt,messagebus,modules_state
import time

modules_state.scopes['x'] = {}
#Create an event that sets y to 0 if it is 1
x = newevt.Event("y==1","global y\ny=0",locals(),setup="y=0")

#Register event with polling.
x.register()
#Set y to 1
x.pymodule.y = 1

time.sleep(0.1)
#y should immediately be set back to 0 at the next polling cycle 
if x.pymodule.y == 1:
        common.fail("Edge-Triggered Event did nothing")
        
#Unregister event and make sure it really goes away
x.unregister() 
x.pymodule.y =1
if not x.pymodule.y == 1:
        common.fail("Edge-Triggered Event did not go away when unregistered")
        
##Testing one time events
x.pymodule.y = False
def f():
    return x.pymodule.y

def g():
    x.pymodule.y = False
    print('anus')
import common
import src.unitsofmeasure as unitsofmeasure
import random

x = unitsofmeasure.timeIntervalFromString("1 minute")
if not x == 60:
    common.fail(x)
    
x = unitsofmeasure.timeIntervalFromString("1 minute 10 seconds")
if not x == 70:
    common.fail(x)
    
x = unitsofmeasure.timeIntervalFromString("1 minute and 10 secondsbutwithanextra42milliseconds")
if not x == 70.042:
    common.fail(x)
    
if not unitsofmeasure.formatTimeInterval(60.1,2) == '1 minute, 100 milliseconds':
    common.fail()
    
if not unitsofmeasure.strToIntWithSIMultipliers('10k') ==10000:
    common.fail()

x = unitsofmeasure.DayOfWeek("Tuesday")
if not x == 1:
    common.fail()
    
if not x == 'tue':
    common.fail()
    
if not x == 'Tuesday':
    common.fail()
cherrypy.HTTPRedirect = BSException

from modules import ActiveModules
wb = modules.WebInterface()

######################################################################################
#Add a new module
try:
   wb.newmoduletarget(name='TEST')
except BSException:
    pass


#Make sure the module shows up in the index
if not 'TEST' in wb.index():
    common.fail("Added new module, but it did not show up in the index")
######################################################################################
#Create a page called testpage
try:
    wb.module('TEST','addresourcetarget','page',name="testpage")
except BSException:
    pass

#View the module's main page and make sure our new page shows up.
if not 'testpage' in wb.module('TEST'):
    common.fail("Made a new module but it didn't show up in the module's index")
###################################################################################### 
#Create a new event
try:
    wb.module('TEST','addresourcetarget','event',name="testevent")
except BSException:
Example #47
0
import newevt
import time
import messagebus

#Create an event that sets y to 0 if it is 1
x = newevt.Event("y==1","y=0",locals())

#Register event with polling.
x.register()
#Set y to 1
y = 1

time.sleep(0.05)
#y should immediately be set back to 0 at the next polling cycle 
if y == 1:
        common.fail("Edge-Triggered Event did nothing")
        
#Unregister event and make sure it really goes away
x.unregister() 
y =1
if not y == 1:
        common.fail("Edge-Triggered Event did not go away when unregistered")
        
        
        
        
#Same exact thing exept we use the onchange
x = newevt.Event("!onchange y","y=5",locals())


#Register event with polling.