def test_delete_files(self): """ Test delete_files() """ Files.write("empty", "file1.txt", TEST_PATH) self.assertTrue(Files.file_exists(TEST_PATH + "file1.txt")) Files.delete_files(TEST_PATH + "file1.txt") self.assertFalse(Files.file_exists(TEST_PATH + "file1.txt"))
def _insert(self): """ Insert records """ if self.debug: print " Creating BD for %s" % self.test_file + FILE_EXT_JSON file_list = [self.test_file + FILE_EXT_JSON] else: file_list = Files.get_file_list(self.generated_folder) for json_file in file_list: if not Files.file_exists(self.generated_folder + json_file): continue data = json.load(open(self.generated_folder + json_file)) sub_tables = OrderedDict() for (_key, _value) in data.items(): # images are not saved in the db if _key == __classmap__[self.name].FIELD_IMAGE: del data[_key] continue if isinstance(_value, dict): sub_tables[_key] = data[_key] del data[_key] del _key self._parse_data("", data, sub_tables)
def test_write_binary(self): """ Test can_include_file() """ Files.write_binary("test", "binary.bin", TEST_PATH) self.assertTrue(Files.file_exists(TEST_PATH + "binary.bin")) content = Files.get_raw_contents(TEST_PATH + "binary.bin") self.assertTrue(content == "74657374") Files.delete_files(TEST_PATH + "binary.bin")
def _import_csv(self): """ Import CSV file to a list """ if not Files.file_exists(self._input_file): return "" with codecs.open(self._input_file, 'rb', encoding='utf-8') as data: self._raw_data = list(csv.reader(data)) self._raw_data.sort(key=operator.itemgetter(0))
def download(self): """ Download HTML web pages """ if self.debug: print " Downloading: %s" % self.test_file + FILE_EXT_HTML url_list = [self.base_url + self.test_file + FILE_EXT_HTML] else: url_list = Files.get_raw_contents(self.name + ".urls", "./") for web_file in url_list: cached_file = web_file.replace(self.base_url, "") if not Files.file_exists(self.download_folder + cached_file): data = Download.download_page(web_file) Files.write(data, cached_file, self.download_folder) elif self.debug: print " File %s already exists" % (self.download_folder + cached_file)
def execute(source, action, binary_file, search): """ Execute the action for a site :param source: the source :param action: the action :param binary_file: the binary/data file :param search: the index/key to search for """ if not Params.valid_action(action): print ERROR_INVALID_ACTION return if not Params.is_valid_source(source): print ERROR_INVALID_SOURCE return if binary_file: if not Files.file_exists(binary_file): print ERROR_FILE_NOT_FOUND % binary_file return if source == "phone": if action == ACTION_READ: from lib.PhoneRead import PhoneRead print PhoneRead.read(binary_file, int(search)) elif action == ACTION_WRITE: from lib.PhoneWrite import PhoneWrite phone = PhoneWrite() phone.save() elif source == "zip_codes": pass if action == ACTION_READ: from lib.ZipCodeRead import ZipCodeRead print ZipCodeRead.read(binary_file, int(search)) elif action == ACTION_WRITE: from lib.ZipCodeWrite import ZipCodeWrite zips = ZipCodeWrite() zips.save()
def download_images(self): """ Download images for a page (listed in the json file) """ if self.debug: print " Downloading Images: %s" % self.test_file + FILE_EXT_JSON file_list = [self.test_file + FILE_EXT_JSON] else: file_list = Files.get_file_list(self.generated_folder) for json_files in file_list: if not Files.file_exists(self.generated_folder + json_files): continue data = json.load(open(self.generated_folder + json_files)) # define the base name for all images base_name = json_files.replace(FILE_EXT_JSON, "") images = __classmap__[self.name].parse_images(base_name, data) for (filename, url) in images.items(): self.download_image(url, filename)
def test_json(self): """ Test jSon files """ if self.debug: print " Testing SQL: %s" % self.test_file + FILE_EXT_JSON file_list = [self.test_file + FILE_EXT_JSON] else: file_list = Files.get_file_list(self.generated_folder) for json_file in file_list: if not Files.file_exists(self.generated_folder + json_file): continue data = json.load(open(self.generated_folder + json_file)) test_data = __classmap__[self.name].get_test_data() ret = self._check_json(data, test_data) if isinstance(ret, str): print " ** TEST Failed for: %s => %s" % (json_file, ret)