def _insert(self):
""" Insert records
"""
if self.debug:
print " Creating BD for %s" % self.test_file + FILE_EXT_JSON
file_list = [self.test_file + FILE_EXT_JSON]
else:
file_list = Files.get_file_list(self.generated_folder)
for json_file in file_list:
if not Files.file_exists(self.generated_folder + json_file):
continue
data = json.load(open(self.generated_folder + json_file))
sub_tables = OrderedDict()
for (_key, _value) in data.items():
# images are not saved in the db
if _key == __classmap__[self.name].FIELD_IMAGE:
del data[_key]
continue
if isinstance(_value, dict):
sub_tables[_key] = data[_key]
del data[_key]
del _key
self._parse_data("", data, sub_tables)
def test_can_exclude_file(self):
""" Test can_include_file()
"""
Files.write("some content", "file1.txt", TEST_PATH)
file_list = Files.get_file_list(TEST_PATH, ["file1.txt"])
self.assertTrue(len(file_list) == 0)
Files.delete_files(TEST_PATH + "file1.txt")
def is_valid_site(site):
""" Check if the site is valid
:param site: The site
:return boolean
"""
sites = Files.get_file_list(SITE_PATH, [".urls"])
return site + ".py" in sites
def test_get_file_list(self):
""" Test get_file_list()
"""
Files.write("empty", "file1.txt", TEST_PATH)
Files.write("empty", "file2.txt", TEST_PATH)
file_list = Files.get_file_list(TEST_PATH)
self.assertTrue(len(file_list) == 2)
self.assertTrue(file_list[1] == "file2.txt")
Files.delete_files(TEST_PATH + "file1.txt")
Files.delete_files(TEST_PATH + "file2.txt")
def download_images(self):
""" Download images for a page (listed in the json file)
"""
if self.debug:
print " Downloading Images: %s" % self.test_file + FILE_EXT_JSON
file_list = [self.test_file + FILE_EXT_JSON]
else:
file_list = Files.get_file_list(self.generated_folder)
for json_files in file_list:
if not Files.file_exists(self.generated_folder + json_files):
continue
data = json.load(open(self.generated_folder + json_files))
# define the base name for all images
base_name = json_files.replace(FILE_EXT_JSON, "")
images = __classmap__[self.name].parse_images(base_name, data)
for (filename, url) in images.items():
self.download_image(url, filename)
def test_json(self):
""" Test jSon files
"""
if self.debug:
print " Testing SQL: %s" % self.test_file + FILE_EXT_JSON
file_list = [self.test_file + FILE_EXT_JSON]
else:
file_list = Files.get_file_list(self.generated_folder)
for json_file in file_list:
if not Files.file_exists(self.generated_folder + json_file):
continue
data = json.load(open(self.generated_folder + json_file))
test_data = __classmap__[self.name].get_test_data()
ret = self._check_json(data, test_data)
if isinstance(ret, str):
print " ** TEST Failed for: %s => %s" % (json_file, ret)