def transfer_file():
    import os
    import tarfile
    import zipfile

    filename = raw_input("Enter the filename you want to move:")
    object_file = raw_input("Enter the object filename:")

    if filename.split(".")[1] == "zip":
        with zipfile.ZipFile(filename) as f_from:
            file_list = f_from.namelist()
            f_from.extractall()
    else:
        with tarfile.open(filename, "r") as f_from:
            file_list = f_from.getnames()
            f_from.extractall()

    if object_file.split(".")[1] == "zip":
        for file in os.listdir(os.getcwd()):
            if file in file_list:
                with zipfile.ZipFile(filename, "a") as f_to:
                    f_to.write(file)
                os.remove(file)
    else:
        for file in os.listdir(os.getcwd()):
            if file in file_list:
                with tarfile.open(object_file, "a") as f_to:
                    f_to.add(file)
                os.remove(file)
Example #2
1
def changeDir(dirname):
    # uplaod form object
    form = forms.UploadForm()
    # user base directory path
    userDir = "uploads/{}".format(current_user.username)
    # if the form is submitted then validate it
    if form.validate_on_submit():
        # create secure filename for uploaded file
        filename = secure_filename(form.upload.data.filename)
        # check if filename exists
        # flash a corresponding message to user
        if filename in listdir("{}/{}".format(userDir, dirname)):
            flash("filename exists", "error")
        else:
            # if file name does not exits then save the file to directory
            form.upload.data.save("{}/{}/{}".format(userDir, dirname, filename))
            flash("file uploaded", "success")
    else:
        # if no form is submitted then set file to none to
        # allow the passing of this variable to the template
        filename = None
        # create a list of filenames in this directory
    files = [f for f in listdir("{}/{}".format(userDir, dirname)) if isfile(join("{}/{}".format(userDir, dirname), f))]
    # render template and pass over variables
    return render_template("directory.html", files=files, form=form, filename=filename, dirname=dirname)
Example #3
1
 def clear(self, in_curr=False):
     for name in os.listdir(self.get_dir()):
         try_delete(os.path.join(self.get_dir(), name) if not in_curr else name)
     emcc_debug = os.environ.get("EMCC_DEBUG")
     if emcc_debug and not in_curr and EMSCRIPTEN_TEMP_DIR:
         for name in os.listdir(EMSCRIPTEN_TEMP_DIR):
             try_delete(os.path.join(EMSCRIPTEN_TEMP_DIR, name))
Example #4
1
    def __init__(
        self,
        marionette_args,
        out_of_process=True,
        profile_data_dir=None,
        locations=os.path.join(here, "server-locations.txt"),
    ):
        super(B2GMochitest, self).__init__()
        self.marionette_args = marionette_args
        self.out_of_process = out_of_process
        self.locations_file = locations
        self.preferences = []
        self.webapps = None
        self.test_script = os.path.join(here, "b2g_start_script.js")
        self.test_script_args = [self.out_of_process]
        self.product = "b2g"

        # structured logging
        self.message_logger = MessageLogger(logger=log)

        if profile_data_dir:
            self.preferences = [
                os.path.join(profile_data_dir, f) for f in os.listdir(profile_data_dir) if f.startswith("pref")
            ]
            self.webapps = [
                os.path.join(profile_data_dir, f) for f in os.listdir(profile_data_dir) if f.startswith("webapp")
            ]

        # mozinfo is populated by the parent class
        if mozinfo.info["debug"]:
            self.SERVER_STARTUP_TIMEOUT = 180
        else:
            self.SERVER_STARTUP_TIMEOUT = 90
Example #5
1
    def build_status(self, args):

        print(
            "{Style.BRIGHT}Bootstraps whose core components are probably "
            "already built:{Style.RESET_ALL}".format(Style=Out_Style)
        )
        for filen in os.listdir(join(self.ctx.build_dir, "bootstrap_builds")):
            print(
                "    {Fore.GREEN}{Style.BRIGHT}{filen}{Style.RESET_ALL}".format(
                    filen=filen, Fore=Out_Fore, Style=Out_Style
                )
            )

        print("{Style.BRIGHT}Recipes that are probably already built:" "{Style.RESET_ALL}".format(Style=Out_Style))
        if exists(join(self.ctx.build_dir, "other_builds")):
            for filen in sorted(os.listdir(join(self.ctx.build_dir, "other_builds"))):
                name = filen.split("-")[0]
                dependencies = filen.split("-")[1:]
                recipe_str = "    {Style.BRIGHT}{Fore.GREEN}{name}" "{Style.RESET_ALL}".format(
                    Style=Out_Style, name=name, Fore=Out_Fore
                )
                if dependencies:
                    recipe_str += (" ({Fore.BLUE}with " + ", ".join(dependencies) + "{Fore.RESET})").format(
                        Fore=Out_Fore
                    )
                recipe_str += "{Style.RESET_ALL}".format(Style=Out_Style)
                print(recipe_str)
Example #6
1
def open_files(nss, gws):
    gofiles = os.listdir(GOPATH)
    nsfiles = os.listdir(NSPATH)
    gwfiles = os.listdir(GWPATH)
    gofp = []
    nsfp = []
    gwfp = []
    for ns in nss:
        if ns in nsfiles:
            nsfp.append(open(NSPATH + ns, "r+"))
        else:
            fp = open(NSPATH + ns, "w")
            fp.write(ns + ",0,0")
            fp.close()
            nsfp.append(open(NSPATH + ns, "r+"))

    for gw in gws:
        if gw in gwfiles:
            gwfp.append(open(GWPATH + ns, "r+"))
        else:
            fp = open(GWPATH + gw, "w")
            fp.write(gw + ",0,0")
            fp.close()
            gwfp.append(open(GWPATH + ns, "r+"))

    if "8.8.8.8" in gofiles:
        gofp.append(open(GOPATH + "8.8.8.8", "r+"))
    else:
        fp = open(GOPATH + "8.8.8.8", "w")
        fp.write("8.8.8.8,0,0")
        fp.close()
        gofp.append(open(GOPATH + "8.8.8.8", "r+"))

    return nsfp, gwfp, gofp
Example #7
1
def main():

    parser = argparse.ArgumentParser(description="quickie")
    parser.add_argument("parentdir", type=str, help="Parent directory of run containing subdirectories of data files")
    args = parser.parse_args()
    pdir = args.parentdir

    # Loop through all subdirectories under the parent directory
    for item in os.listdir(pdir):
        if os.path.isdir(os.path.join(pdir, item)):
            # if OUTPUT exists, cd into it and begin plotting
            if "OUTPUT" in os.listdir(os.path.join(pdir, item)):
                print "Found output"
                os.chdir(os.path.join(pdir, item) + "/OUTPUT")
                try:
                    os.mkdir("donotdelete")
                except OSError:
                    None
                supers = glob.glob("gce-super*")
                for supfile in supers:
                    ID = supfile[-13:]
                    newfiles = glob.glob("*%s" % ID)
                    for newfile in newfiles:
                        sh("mv %s donotdelete" % newfile)
                sh("rm *.dat")
                os.chdir(pdir)
            else:
                None
Example #8
0
    def __locateRequest(self, requestName, assigned=False):
        """ Locate the sub requests associated with a requestName

    :param self: self reference
    :param str requestName: request name
    :param bool assigned: flag to include/exclude Assigned requests
    """
        self.log.info("__locateRequest: Attempting to locate %s." % requestName)
        requestTypes = os.listdir(self.root)
        subRequests = []
        try:
            for requestType in requestTypes:
                reqDir = "%s/%s" % (self.root, requestType)
                if os.path.isdir(reqDir):
                    statusList = os.listdir(reqDir)
                    if not assigned and "Assigned" in statusList:
                        statusList.remove("Assigned")
                    for status in statusList:
                        statusDir = os.path.join(reqDir, status)
                        if os.path.isdir(statusDir):
                            requestNames = os.listdir(statusDir)
                            if requestName in requestNames:
                                requestPath = os.path.join(statusDir, requestName)
                                subRequests.append(requestPath)
            self.log.info("__locateRequest: Successfully located %s." % requestName)
            return S_OK(subRequests)
        except Exception, error:
            errStr = "__locateRequest: Exception while locating request."
            self.log.exception(errStr, requestName, lException=error)
            return S_ERROR(errStr)
Example #9
0
 def test_checkout_alternative_dir(self):
     ref_i18n = self.repo.lookup_reference("refs/heads/i18n")
     extra_dir = os.path.join(self.repo.workdir, "extra-dir")
     os.mkdir(extra_dir)
     self.assertTrue(len(os.listdir(extra_dir)) == 0)
     self.repo.checkout(ref_i18n, directory=extra_dir)
     self.assertFalse(len(os.listdir(extra_dir)) == 0)
 def install(self):
     curr_dir = os.getcwd()
     tempdir = tempfile.mkdtemp()
     try:
         filename = os.path.join(tempdir, "libevent.tar.gz")
         logger.info("Downloading libevent source from %s", self.libevent_source)
         urllib.urlretrieve(self.libevent_source, filename)
         assert os.path.isfile(filename)
         os.chdir(tempdir)
         logger.info("Extracting libevent...")
         contents_before = set(os.listdir(tempdir))
         ret = subprocess.call(["tar", "xzf", filename])
         assert ret == 0
         new_contents = list(contents_before.symmetric_difference(os.listdir(tempdir)))
         assert len(new_contents) == 1
         libevent_root = os.path.join(tempdir, new_contents[0])
         os.chdir(libevent_root)
         logger.info("Compiling libevent...")
         ret = subprocess.call(["sh", "./configure", "--with-pic"])
         assert ret == 0
         ret = subprocess.call(["make", "-j", "4"])
         assert ret == 0
         os.environ["LIBEVENT_ROOT"] = libevent_root
         return super(Recipe, self).install()
     finally:
         os.chdir(curr_dir)
         shutil.rmtree(tempdir)
Example #11
0
 def test_compile(self, tmpdir, command, filename):
     tmpdir = str(tmpdir)
     self.create_config(8, 0.1, tmpdir)
     assert os.listdir(tmpdir) == ["pwhash.json"]
     with self.spawn(command, cwd=tmpdir) as p:
         p.expect("%r created!" % filename)
     assert set(os.listdir(tmpdir)) == {"pwhash.json", filename}
Example #12
0
def find_lock():
    from os.path import join

    from conda.lock import LOCKFN

    lock_dirs = config.pkgs_dirs[:]
    lock_dirs += [config.root_dir]
    for envs_dir in config.envs_dirs:
        if os.path.exists(envs_dir):
            for fn in os.listdir(envs_dir):
                if os.path.isdir(join(envs_dir, fn)):
                    lock_dirs.append(join(envs_dir, fn))

    try:
        from conda_build.config import croot

        lock_dirs.append(croot)
    except ImportError:
        pass

    for dir in lock_dirs:
        if not os.path.exists(dir):
            continue
        for dn in os.listdir(dir):
            if os.path.isdir(join(dir, dn)) and dn.startswith(LOCKFN):
                path = join(dir, dn)
                yield path
Example #13
0
def choose_plugins_to_enable() -> list:
    """
    Seeks all findable plugins and asks which ones to enable

    :return list of plugins to enable
    """
    plugins_to_enable = list()

    for package in os.listdir(constants.PLUGINS_PATH):
        package_full_path = os.path.join(constants.PLUGINS_PATH, package)
        if not os.path.isdir(package_full_path):
            continue

        for _file_ in os.listdir(package_full_path):
            if _file_.endswith(".py") and _file_ != "__init__.py":
                plugin = "{}.{}".format(package, os.path.splitext(_file_)[0])
                plugin_is_enabled = True if plugin in get_global_conf().get("plugins", "enabled_plugins") else False
                answer = ask_question(
                    "Would you like to enable the {} plugin ? {}".format(
                        plugin, format_closed_question(plugin_is_enabled)
                    ),
                    validate_answers=validate_closed_question,
                    default="y" if plugin_is_enabled else "n"
                )
                if answer:
                    plugins_to_enable.append(plugin)

    return plugins_to_enable
Example #14
0
    def install_precompiled(self):
        """Installs this project's precompiled code into MCP's classes.

           This code will not be included in the project's package, as it's
           assumed to be libraries or similar code needed for reobfuscation,
           but not part of the mod itself.

           I use this with a deobfuscated-but-still-compiled copy of
           IC2 (which I compile against) so that it reobfuscates happily
           without having to actually solve decompilation issues.
        """
        did_something = False

        bin = os.path.join(self.dir, "bin")
        if os.path.isdir(bin):
            # Common classes into both sides first, so it can be overridden.
            common = os.path.join(bin, "common")
            if os.path.isdir(common) and os.listdir(common):
                self.copy_files(common, MCP_BIN_CLIENT, BIN_INSTALL_FAILED)
                self.copy_files(common, MCP_BIN_SERVER, BIN_INSTALL_FAILED)
                did_something = True

            # Then client classes.
            client = os.path.join(bin, "client")
            if os.path.isdir(client) and os.listdir(client):
                self.copy_files(client, MCP_BIN_CLIENT, BIN_INSTALL_FAILED)
                did_something = True

            # And finally server classes.
            server = os.path.join(bin, "server")
            if os.path.isdir(server) and os.listdir(server):
                self.copy_files(server, MCP_BIN_SERVER, BIN_INSTALL_FAILED)
                did_something = True

        return did_something
Example #15
0
    def install(self):
        """Installs this project into MCP's source."""
        did_something = False

        src = os.path.join(self.dir, "src")
        if os.path.isdir(src):
            # Common code into both sides first, so it can be overridden.
            common = os.path.join(src, "common")
            if os.path.isdir(common) and os.listdir(common):
                self.copy_files(common, MCP_SRC_CLIENT, SRC_INSTALL_FAILED)
                self.copy_files(common, MCP_SRC_SERVER, SRC_INSTALL_FAILED)
                did_something = True

            # Then client code.
            client = os.path.join(src, "client")
            if os.path.isdir(client) and os.listdir(client):
                self.copy_files(client, MCP_SRC_CLIENT, SRC_INSTALL_FAILED)
                did_something = True

            # And finally server code.
            server = os.path.join(src, "server")
            if os.path.isdir(server) and os.listdir(server):
                self.copy_files(server, MCP_SRC_SERVER, SRC_INSTALL_FAILED)
                did_something = True

        return did_something
def pos_eng(path_load, path_pos, path_word_pos):
    for grade in os.listdir(path_load):
        if grade != ".DS_Store":
            path_to_grade = path_load + grade + "/"
            for filename in os.listdir(path_to_grade):
                if filename != ".DS_Store":
                    with open(path_to_grade + filename, "r") as my_file:
                        data = my_file.read().replace("\n", "")
                        data = data.decode("utf-8")

                    text_pos = []
                    text_word_pos = []
                    for sent in nltk.sent_tokenize(data):
                        for chunk in nltk.pos_tag(nltk.word_tokenize(sent)):
                            if isinstance(chunk, tuple):
                                text_pos.append(chunk[1])
                                text_word_pos.append(chunk[0] + "_" + chunk[1])

                    if not os.path.exists(os.path.dirname(path_pos + grade + "/" + filename)):
                        os.makedirs(os.path.dirname(path_pos + grade + "/" + filename))
                    pos_file = open(path_pos + grade + "/" + filename, "w+")

                    for word in text_pos:
                        pos_file.write("%s " % word)
                    pos_file.close()

                    if not os.path.exists(os.path.dirname(path_word_pos + grade + "/" + filename)):
                        os.makedirs(os.path.dirname(path_word_pos + grade + "/" + filename))
                    word_pos_file = open(path_word_pos + grade + "/" + filename, "w+")

                    for word in text_word_pos:
                        word_pos_file.write("%s " % word.encode("utf-8"))
                    word_pos_file.close()
def ig_eng(path_load, path_ig):
    grades = os.listdir(path_load)
    grades.remove(".DS_Store")

    ig_words = get_all_ig(path_load, grades)

    for grade in grades:
        path_to_grade = path_load + grade + "/"
        for filename in os.listdir(path_to_grade):
            if filename != ".DS_Store":
                with open(path_to_grade + filename, "r") as my_file:
                    data = my_file.read().replace("\n", "")
                    data = data.decode("utf-8")

                text = []
                for sent in nltk.sent_tokenize(data):
                    for chunk in nltk.pos_tag(nltk.word_tokenize(sent)):
                        if isinstance(chunk, tuple):
                            if chunk[0] in ig_words:
                                text.append(chunk[0])
                            else:
                                text.append(chunk[1])

                if not os.path.exists(os.path.dirname(path_ig + grade + "/" + filename)):
                    os.makedirs(os.path.dirname(path_ig + grade + "/" + filename))

                ig_file = open(path_ig + grade + "/" + filename, "w+")

                for word in text:
                    ig_file.write("%s " % word.encode("utf-8"))
                ig_file.close()
Example #18
0
    def get_config_files():
        """
        Return a list of configuration files describing the apps supported by
        Mackup. The files return are absolute fullpath to those files.
        e.g. /usr/lib/mackup/applications/bash.cfg

        Returns:
            set of strings.
        """
        # Configure the config parser
        apps_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), APPS_DIR)
        custom_apps_dir = os.path.join(os.environ["HOME"], CUSTOM_APPS_DIR)

        # Build the list of stock application config files
        config_files = set()
        for filename in os.listdir(apps_dir):
            if filename.endswith(".cfg"):
                config_files.add(os.path.join(apps_dir, filename))

        # Append the list of custom application config files
        if os.path.isdir(custom_apps_dir):
            for filename in os.listdir(custom_apps_dir):
                if filename.endswith(".cfg"):
                    config_files.add(os.path.join(custom_apps_dir, filename))
        return config_files
def train_load():
    path = "data/train"

    Dir = listdir(path)
    imagelist = []
    ListOfName = []
    count = 0
    for directory in Dir:
        if directory != ".DS_Store":  # For some reason this directory keep bumping up
            subpath = listdir(path + "/" + directory)
            ListOfName.append(directory)
            print(directory)
            count = count + 1
            for subdir in subpath:
                if subdir != ".DS_Store":
                    Image = image(path + "/" + directory + "/" + subdir, count)
                    Image = Image.get_feature(reshape=True)
                    # Image = Image.getGaborFeatures(reshape = True)
                    output = count - 1
                    #        output = numpy.zeros((121,1))
                    #       output[count -1] = numpy.float64(1)
                    imagelist.append([Image, output])

    # we need to randomly shuffle the data to retain the generality
    shuffle(imagelist)
    with open("pickle/ListOfName.pkl", "w") as f:
        pickle.dump(ListOfName, f)
    print("got image list for train")
    return imagelist, count
Example #20
0
def main():
    try:
        global End
        global Flag
        os.system("rm -rf " + ConstPath + "/allpairs/negative/*~ &>/dev/null")
        os.system("rm -rf " + ConstPath + "/allpairs/positive/*~ &>/dev/null")
        os.system("rm -rf " + ConstPath + "/allpairs/positive/case*txt &>/dev/null")
        os.system("rm -rf " + ConstPath + "/tools/crosswalk/*apk &>/dev/null")
        os.system("rm -rf " + ConstPath + "/self &>/dev/null")
        os.system("mkdir -p " + ConstPath + "/self")
        devicesConform()

        # positive test
        for seed in os.listdir(ConstPath + "/allpairs/positive/"):
            os.system("rm -rf " + ConstPath + "/allpairs/selfcomb.txt &>/dev/null")
            os.system("rm -rf " + ConstPath + "/self &>/dev/null")
            os.system("mkdir -p " + ConstPath + "/self")
            processMain(ConstPath + "/allpairs/positive/" + seed)

        # negative case
        Flag = "negative"
        for seed in os.listdir(ConstPath + "/allpairs/negative/"):
            os.system("rm -rf " + ConstPath + "/allpairs/selfcomb.txt &>/dev/null")
            os.system("rm -rf " + ConstPath + "/self &>/dev/null")
            os.system("mkdir -p " + ConstPath + "/self")
            processMain(ConstPath + "/allpairs/negative/" + seed)

        End = time.strftime("%Y-%m-%d %H:%M:%S")
        genResultXml()
        genSummaryXml()
    except Exception, e:
        print Exception, ":", e
        sys.exit(1)
Example #21
0
    def getDBSummary(self):
        """ Obtain a summary of the contents of the requestDB

    :param self: self reference
    :return: S_OK with dict[requestType][status] => nb of files
    """
        self.log.info("getDBSummary: Attempting to get database summary.")
        requestTypes = os.listdir(self.root)
        try:
            summaryDict = {}
            for requestType in requestTypes:
                summaryDict[requestType] = {}
                reqTypeDir = os.path.join(self.root, requestType)
                if os.path.isdir(reqTypeDir):
                    statusList = os.listdir(reqTypeDir)
                    for status in statusList:
                        reqTypeStatusDir = os.path.join(reqTypeDir, status)
                        requests = os.listdir(reqTypeStatusDir)
                        summaryDict[requestType][status] = len(requests)
            self.log.info("getDBSummary: Successfully obtained database summary.")
            return S_OK(summaryDict)
        except Exception, x:
            errStr = "getDBSummary: Exception while getting DB summary."
            self.log.exception(errStr, lException=x)
            return S_ERROR(errStr)
Example #22
0
 def _assert_tar_count_equals(self, file_name, count):
     if not tarfile.is_tarfile(file_name):
         # Mabye it's not a tar cuz it's a status message.
         fin = open(file_name, "r")
         contents = fin.read(256)
         fin.close()
         if contents.lower().find("no ") != -1:
             self.assertEqual(0, count)
             return
         raise Exception("%s is not a tar file" % file_name)
     tar = tarfile.open(file_name)
     tmp_dir = "unit_test_tmp"
     if os.path.exists(tmp_dir):
         filenames = os.listdir(tmp_dir)
         for file in filenames:
             os.remove(os.path.join(tmp_dir, file))
         os.rmdir(tmp_dir)
     os.mkdir(tmp_dir)
     tar.extractall(path=tmp_dir)
     tar.close()
     filenames = os.listdir(tmp_dir)
     try:
         self.assertEqual(len(filenames), count)
     finally:
         # clean up
         for file in filenames:
             os.remove(os.path.join(tmp_dir, file))
         os.rmdir(tmp_dir)
Example #23
0
def iter_dir(loc):
    for fn in os.listdir(loc):
        if path.isdir(path.join(loc, fn)):
            for sub in os.listdir(path.join(loc, fn)):
                yield path.join(loc, fn, sub)
        else:
            yield path.join(loc, fn)
Example #24
0
 def _data_set(self, cr, obj, id, name, value, uid=None, context={}):
     if not value:
         return True
     if (not context) or context.get("store_method", "fs") == "fs":
         path = os.path.join(os.getcwd(), "filestore")
         if not os.path.isdir(path):
             os.mkdir(path)
         flag = None
         for dirs in os.listdir(path):
             if os.path.isdir(os.path.join(path, dirs)) and len(os.listdir(os.path.join(path, dirs))) < 4000:
                 flag = dirs
                 break
         flag = flag or create_directory(path)
         filename = random_name()
         fname = os.path.join(path, flag, filename)
         fp = file(fname, "wb")
         fp.write(base64.decodestring(value))
         filesize = os.stat(fname).st_size
         cr.execute(
             "update ir_attachment set store_fname=%s,store_method=%s,file_size=%d where id=%d",
             (os.path.join(flag, filename), "fs", filesize, id),
         )
     else:
         cr.execute(
             "update ir_attachment set datas=%s,store_method=%s where id=%d", (psycopg.Binary(value), "db", id)
         )
     return True
Example #25
0
    def compareState(self):
        pathToStateData = "./state/"

        # initial first png and xml
        # shutil.copy('0.png', pathToStateData + str(0) + '.png')
        # shutil.copy('0.xml', pathToStateData + str(0) + '.xml')
        imgFiles = [name for name in os.listdir(pathToStateData) if name.endswith(".png")]
        xmlFiles = [name for name in os.listdir(pathToStateData) if name.endswith(".xml")]
        txtFiles = [name for name in os.listdir("./state/") if name.endswith(".txt")]

        listCheckState = []
        for i in xrange(len(imgFiles)):
            listCheckState.append(calc.pixelCompare("0.png", pathToStateData + imgFiles[i], 0.1))
        print "check state: " + str(listCheckState)
        # all -> and list, any or list
        if all(listCheckState) == True:
            for i in xrange(len(imgFiles) + 1):
                if str(i) + ".png" not in imgFiles:
                    shutil.copy("0.png", pathToStateData + str(i) + ".png")
                    shutil.copy("0.xml", pathToStateData + str(i) + ".xml")
                    shutil.copy("xmlcv.txt", "./state/" + str(i) + ".txt")
                    print "state: " + str(i)
        else:
            for i, j in enumerate(listCheckState):
                if j == False:
                    print "state: " + str(i)
                    break
        """
def load_letter(folder, min_num_images):
    """Load the data for a single letter label."""
    image_files = os.listdir(folder)
    dataset = np.ndarray(shape=(len(image_files), image_size, image_size), dtype=np.float32)
    image_index = 0
    print(folder)
    for image in os.listdir(folder):
        image_file = os.path.join(folder, image)
        try:
            image_data = (ndimage.imread(image_file).astype(float) - pixel_depth / 2) / pixel_depth
            if image_data.shape != (image_size, image_size):
                raise Exception("Unexpected image shape: %s" % str(image_data.shape))
            dataset[image_index, :, :] = image_data
            image_index += 1
        except IOError as e:
            print("Could not read:", image_file, ":", e, "- it's ok, skipping.")

    num_images = image_index
    dataset = dataset[0:num_images, :, :]
    if num_images < min_num_images:
        raise Exception("Many fewer images than expected: %d < %d" % (num_images, min_num_images))

    print("Full dataset tensor:", dataset.shape)
    print("Mean:", np.mean(dataset))
    print("Standard deviation:", np.std(dataset))
    return dataset
Example #27
0
    def enumerate_modules(self, show_all=False):
        """
        *Availability: 4.0+*

        Return a dict mapping the names of modules to the location of their
        file.  This searches the regular modules directory and all directories
        specified in the `core.extra` attribute of the `config` object. If two
        modules have the same name, the last one to be found will be returned
        and the rest will be ignored. Modules are found starting in the regular
        directory, followed by `~/.willie/modules`, and then through the extra
        directories in the order that the are specified.

        If `show_all` is given as `True`, the `enable` and `exclude`
        configuration options will be ignored, and all modules will be shown
        (though duplicates will still be ignored as above).
        """
        modules = {}

        # First, add modules from the regular modules directory
        this_dir = os.path.dirname(os.path.abspath(__file__))
        modules_dir = os.path.join(this_dir, "modules")
        for fn in os.listdir(modules_dir):
            if fn.endswith(".py") and not fn.startswith("_"):
                modules[fn[:-3]] = os.path.join(modules_dir, fn)
        # Next, look in ~/.willie/modules
        home_modules_dir = os.path.join(os.path.expanduser("~"), ".willie", "modules")
        if not os.path.isdir(home_modules_dir):
            os.makedirs(home_modules_dir)
        for fn in os.listdir(home_modules_dir):
            if fn.endswith(".py") and not fn.startswith("_"):
                modules[fn[:-3]] = os.path.join(home_modules_dir, fn)

        # Last, look at all the extra directories. (get_list returns [] if
        # there are none or the option isn't defined, so it'll just skip this
        # bit)
        for directory in self.core.get_list("extra"):
            for fn in os.listdir(directory):
                if fn.endswith(".py") and not fn.startswith("_"):
                    modules[fn[:-3]] = os.path.join(directory, fn)

        # If caller wants all of them, don't apply white and blacklists
        if show_all:
            return modules

        # Apply whitelist, if present
        enable = self.core.get_list("enable")
        if enable:
            enabled_modules = {}
            for module in enable:
                if module in modules:
                    enabled_modules[module] = modules[module]
            modules = enabled_modules

        # Apply blacklist, if present
        exclude = self.core.get_list("exclude")
        for module in exclude:
            if module in modules:
                del modules[module]

        return modules
    def _normalize_article_dir_with_dtd(self, path):
        """
        TODO: main.xml from Springer assume the existence of a local DTD.
        This procedure install the DTDs next to the main.xml file
        and normalize it using xmllint in order to resolve all namespaces
        and references.
        """
        files = [filename for filename in listdir(path) if "nlm.xml" in filename]
        if not files:
            files = [filename for filename in listdir(path) if ".xml.scoap" in filename]
        if exists(join(path, "resolved_main.xml")):
            return

        if "JATS-archivearticle1.dtd" in open(join(path, files[0])).read():
            path_normalized = mkdtemp(prefix="scoap3_normalized_jats_", dir=CFG_TMPSHAREDDIR)
            ZipFile(CFG_SPRINGER_JATS_PATH).extractall(path_normalized)
        elif "A++V2.4.dtd" in open(join(path, files[0])).read():
            path_normalized = mkdtemp(prefix="scoap3_normalized_app_", dir=CFG_TMPSHAREDDIR)
            ZipFile(CFG_SPRINGER_AV24_PATH).extractall(path_normalized)
        else:
            error_msg = (
                "It looks like the path %s does not contain an " "JATS-archivearticle1.dtd nor A++V2.4.dtd XML file."
            )
            self.logger.error(error_msg % path)
            raise ValueError(error_msg % path)
        print "Normalizing %s" % (files[0],)
        (cmd_exit_code, cmd_out, cmd_err) = run_shell_command(
            ("xmllint --format " "--loaddtd %s --output %s"),
            (join(path, files[0]), join(path_normalized, "resolved_main.xml")),
        )
        if cmd_err:
            error_msg = "Error in cleaning %s: %s"
            self.logger.error(error_msg % (join(path, "issue.xml"), cmd_err))
            raise ValueError(error_msg % (join(path, "main.xml"), cmd_err))
        self.articles_normalized.append(path_normalized)
Example #29
0
    def cardsinfo(self, args, options):
        """Basically a -incomplete- reimplementation of dahdi_hardware program
        """
        devices = {}
        DEVPATH = "/sys/bus/pci/devices"

        for pciid in os.listdir(DEVPATH):
            dev = {"loaded": False}
            for k in ("vendor", "device", "subsystem_vendor", "subsystem_device"):
                with open(os.path.join(DEVPATH, pciid, k)) as f:
                    dev[k] = f.read()[2:-1]  #  remove starting '0x' and trailing '\n'

            pcikey = re.split("([:/])", "{vendor}:{device}/{subsystem_vendor}:{subsystem_device}".format(**dev))
            for i in (7, 5, 3):
                subkey = "".join(pcikey[:i])
                if subkey in PCI_IDS:
                    dev.update(PCI_IDS[subkey])
                    devices[pciid] = dev
                    break

        DRVPATH = "/sys/bus/pci/drivers"
        for drvname in os.listdir(DRVPATH):
            try:
                #  fail if no pciid (i.e 0000:03:0c.0) found
                pciid = [f for f in os.listdir(os.path.join(DRVPATH, drvname)) if re.match("^[\da-f.:]+$", f)][0]

                # fail if no module defined for pci device
                modname = os.path.basename(os.readlink(os.path.join(DRVPATH, drvname, "module")))

                #  fail if pci device is unknown
                devices[pciid]["loaded"] = devices[pciid]["driver"] == modname
            except:
                continue

        return devices
Example #30
0
    def test_noclean(self):
        """test `restore=True/False` functionality"""

        profile = tempfile.mkdtemp()
        tmpdir = tempfile.mkdtemp()
        try:

            # empty initially
            self.assertFalse(bool(os.listdir(profile)))

            # make an addon
            stub = addon_stubs.generate_addon(name="empty-0-1.xpi", path=tmpdir)

            # install it with a restore=True AddonManager
            addons = mozprofile.addons.AddonManager(profile, restore=True)
            addons.install_from_path(stub)

            # now its there
            self.assertEqual(os.listdir(profile), ["extensions"])
            extensions = os.path.join(profile, "extensions", "staged")
            self.assertTrue(os.path.exists(extensions))
            contents = os.listdir(extensions)
            self.assertEqual(len(contents), 1)

            # del addons; now its gone though the directory tree exists
            del addons
            self.assertEqual(os.listdir(profile), ["extensions"])
            self.assertTrue(os.path.exists(extensions))
            contents = os.listdir(extensions)
            self.assertEqual(len(contents), 0)

        finally:
            mozfile.rmtree(tmpdir)
            mozfile.rmtree(profile)