def test_delete_files(self):
""" Test delete_files()
"""
Files.write("empty", "file1.txt", TEST_PATH)
self.assertTrue(Files.file_exists(TEST_PATH + "file1.txt"))
Files.delete_files(TEST_PATH + "file1.txt")
self.assertFalse(Files.file_exists(TEST_PATH + "file1.txt"))
def _insert(self):
""" Insert records
"""
if self.debug:
print " Creating BD for %s" % self.test_file + FILE_EXT_JSON
file_list = [self.test_file + FILE_EXT_JSON]
else:
file_list = Files.get_file_list(self.generated_folder)
for json_file in file_list:
if not Files.file_exists(self.generated_folder + json_file):
continue
data = json.load(open(self.generated_folder + json_file))
sub_tables = OrderedDict()
for (_key, _value) in data.items():
# images are not saved in the db
if _key == __classmap__[self.name].FIELD_IMAGE:
del data[_key]
continue
if isinstance(_value, dict):
sub_tables[_key] = data[_key]
del data[_key]
del _key
self._parse_data("", data, sub_tables)
def test_write_binary(self):
""" Test can_include_file()
"""
Files.write_binary("test", "binary.bin", TEST_PATH)
self.assertTrue(Files.file_exists(TEST_PATH + "binary.bin"))
content = Files.get_raw_contents(TEST_PATH + "binary.bin")
self.assertTrue(content == "74657374")
Files.delete_files(TEST_PATH + "binary.bin")
def _import_csv(self):
""" Import CSV file to a list
"""
if not Files.file_exists(self._input_file):
return ""
with codecs.open(self._input_file, 'rb', encoding='utf-8') as data:
self._raw_data = list(csv.reader(data))
self._raw_data.sort(key=operator.itemgetter(0))
def download(self):
""" Download HTML web pages
"""
if self.debug:
print " Downloading: %s" % self.test_file + FILE_EXT_HTML
url_list = [self.base_url + self.test_file + FILE_EXT_HTML]
else:
url_list = Files.get_raw_contents(self.name + ".urls", "./")
for web_file in url_list:
cached_file = web_file.replace(self.base_url, "")
if not Files.file_exists(self.download_folder + cached_file):
data = Download.download_page(web_file)
Files.write(data, cached_file, self.download_folder)
elif self.debug:
print " File %s already exists" % (self.download_folder +
cached_file)
def execute(source, action, binary_file, search):
""" Execute the action for a site
:param source: the source
:param action: the action
:param binary_file: the binary/data file
:param search: the index/key to search for
"""
if not Params.valid_action(action):
print ERROR_INVALID_ACTION
return
if not Params.is_valid_source(source):
print ERROR_INVALID_SOURCE
return
if binary_file:
if not Files.file_exists(binary_file):
print ERROR_FILE_NOT_FOUND % binary_file
return
if source == "phone":
if action == ACTION_READ:
from lib.PhoneRead import PhoneRead
print PhoneRead.read(binary_file, int(search))
elif action == ACTION_WRITE:
from lib.PhoneWrite import PhoneWrite
phone = PhoneWrite()
phone.save()
elif source == "zip_codes":
pass
if action == ACTION_READ:
from lib.ZipCodeRead import ZipCodeRead
print ZipCodeRead.read(binary_file, int(search))
elif action == ACTION_WRITE:
from lib.ZipCodeWrite import ZipCodeWrite
zips = ZipCodeWrite()
zips.save()
def download_images(self):
""" Download images for a page (listed in the json file)
"""
if self.debug:
print " Downloading Images: %s" % self.test_file + FILE_EXT_JSON
file_list = [self.test_file + FILE_EXT_JSON]
else:
file_list = Files.get_file_list(self.generated_folder)
for json_files in file_list:
if not Files.file_exists(self.generated_folder + json_files):
continue
data = json.load(open(self.generated_folder + json_files))
# define the base name for all images
base_name = json_files.replace(FILE_EXT_JSON, "")
images = __classmap__[self.name].parse_images(base_name, data)
for (filename, url) in images.items():
self.download_image(url, filename)
def test_json(self):
""" Test jSon files
"""
if self.debug:
print " Testing SQL: %s" % self.test_file + FILE_EXT_JSON
file_list = [self.test_file + FILE_EXT_JSON]
else:
file_list = Files.get_file_list(self.generated_folder)
for json_file in file_list:
if not Files.file_exists(self.generated_folder + json_file):
continue
data = json.load(open(self.generated_folder + json_file))
test_data = __classmap__[self.name].get_test_data()
ret = self._check_json(data, test_data)
if isinstance(ret, str):
print " ** TEST Failed for: %s => %s" % (json_file, ret)
