def travis_permissions(cls):
"""
Set permissions in order to avoid issues before commiting.
"""
try:
build_dir = environ['TRAVIS_BUILD_DIR']
commands = [
'sudo chown -R travis:travis %s' % (build_dir),
'sudo chgrp -R travis %s' % (build_dir),
'sudo chmod -R g+rwX %s' % (build_dir),
'sudo chmod 777 -Rf %s.git' %
(build_dir + directory_separator),
r"sudo find %s -type d -exec chmod g+x '{}' \;" % (build_dir)
]
for command in commands:
Helpers.Command(command).execute()
if Helpers.Command(
'git config core.sharedRepository').execute() == '':
Helpers.Command(
'git config core.sharedRepository group').execute()
except NameError:
pass
return
def get_valid_extensions(self):
"""
Get the list of valid extensions based on the result of self.download().
"""
result = {}
regex_valid_extension = r'(/domains/root/db/)(.*)(\.html)'
for readed in open(self.download_destination):
readed = readed.rstrip('\n').strip()
matched = Helpers.Regex(readed,
regex_valid_extension,
return_data=True,
rematch=True).match()
if not matched:
continue
else:
ext_with_referer = {matched[1]: self.referer(matched[1])}
result.update(ext_with_referer)
Helpers.File(self.download_destination).delete()
Helpers.File(self.destination).delete()
return result
def __init__(self,
file_to_install=None,
data_to_install=None,
production=False):
Check()
_path = getcwd()
if not _path.endswith(directory_separator):
_path += directory_separator
self.path = _path
if file_to_install is None:
self.file_to_install = 'PyFunceble.py'
else:
self.file_to_install = file_to_install
self.file_to_install = self.path + self.file_to_install
self.production = production
self.data_to_install = data_to_install
if self.production and not Settings.quiet:
print('\nDefault timeout: %s seconds' %
self.default_values()['seconds_before_http_timeout'])
print('Installation of default variables for production', end=" ")
else:
if not Settings.quiet:
print('\n\nInstallation of working directory', end=" ")
self.execute()
regex_skip = r'\[PyFunceble\sskip\]|\[ci\sskip\]'
if Helpers.Regex(Helpers.Command('git log -2').execute(),
regex_skip,
return_data=False).match() or self.production:
Clean()
Directory(self.production)
if self.production and not Settings.quiet:
print(Fore.CYAN + Style.BRIGHT +
'\n\nThe production logic was successfully completed!')
print('You can now distribute this repository.\n')
else:
if not Settings.quiet:
print(Fore.CYAN + Style.BRIGHT +
'\n\nThe installation was successfully completed!')
print(
"You can now use the script with '%s' or learn how to use it with '%s'\n"
% # pylint: disable=line-too-long
(Style.BRIGHT + './' + Settings.script + '.py [-OPTIONS]' +
Style.RESET_ALL, Style.BRIGHT + './' + Settings.script +
'.py --help' + Style.RESET_ALL))
def git(cls):
"""
Update repository if cloned (git).
"""
if Settings.stable:
Helpers.Command('git checkout master').execute()
else:
Helpers.Command('git checkout dev').execute()
print(Helpers.Command('git pull').execute())
return
def them_all(self):
"""
Delete all discovered files.
"""
to_delete = self.file_to_delete()
for file in to_delete:
Helpers.File(file).delete()
def __init__(self):
self.current_path = getcwd()
self.destination = self.current_path + \
directory_separator + Settings.funilrys + '.'
self.files = {
'script': 'PyFunceble.py',
'tool': 'tool.py',
'iana': 'iana-domains-db.json',
'dir_structure': 'dir_structure.json'
}
if path.isdir(self.current_path + directory_separator +
'.git') and Settings.script in Helpers.Command(
'git remote show origin').execute():
self.git()
else:
if not self.same_version(True):
for data in self.files:
Helpers.File(self.current_path + directory_separator +
self.files[data]).delete()
rename(
self.destination + self.files[data],
self.current_path + directory_separator +
self.files[data])
if not Settings.quiet:
print('Checking version', end=' ')
if self.same_version() and not Settings.quiet:
print(Settings.done +
'\n\nThe update was successfully completed!')
else:
if not Settings.quiet:
print(
Settings.error +
'\nImpossible to update PyFunceble. Please report issue.'
)
else:
if not Settings.quiet:
print('No need to update.\n')
for data in self.files:
Helpers.File(self.destination + self.files[data]).delete()
def restore_replace(self):
"""
Check if we need to replace ".gitignore" to ".keep".
"""
if path.isdir(self.base + '.git'):
if Settings.script not in \
Helpers.Command('git remote show origin').execute():
return True
return False
return True
def restore(self):
"""
Restore the 'output/' directory structure based on the `dir_structure.json` file.
"""
if not Settings.quiet:
print('Creation of non existant files and directories', end=" ")
structure = Helpers.Dict().from_json(
Helpers.File(self.structure).read())
structure = structure['output']
replace = self.restore_replace()
for directory in structure:
if not path.isdir(self.base + self.path + directory):
self.travis_permissions()
mkdir(self.base + self.path + directory)
self.travis_permissions()
for file in structure[directory]:
file_path = self.path + directory + directory_separator + file
content_to_write = structure[directory][file]['content']
online_sha = structure[directory][file]['sha512']
content_to_write = Helpers.Regex(content_to_write,
'@@@',
escape=True,
replace_with='\\n').replace()
git_to_keep = file_path.replace('gitignore', 'keep')
keep_to_git = file_path.replace('keep', 'gitignore')
if replace:
if path.isfile(file_path) and Hash(
file_path, 'sha512', True).get() == online_sha:
rename(file_path, git_to_keep)
write = False
else:
Helpers.File(file_path).delete()
file_path = git_to_keep
write = True
else:
if path.isfile(keep_to_git) and Hash(
file_path, 'sha512', True).get() == online_sha:
rename(file_path, keep_to_git)
write = False
else:
Helpers.File(keep_to_git).delete()
file_path = keep_to_git
write = True
if write:
Helpers.File(file_path).write(content_to_write + '\n',
True)
if not Settings.quiet:
print(Settings.done)
def update(self):
"""
Update the content of the `iana-domains-db` file.
"""
if self.download():
Helpers.Dict(self.get_valid_extensions()).to_json(self.destination)
# for extension in extensions:
# Helpers.File(self.destination).write(extension + '\n')
if not Settings.quiet:
print(Settings.done)
else:
if not Settings.quiet:
print(Settings.error)
exit(1)
def backup(self):
"""
Backup the developer state of `output/` in order to make it restorable
and portable for user.
"""
result = {'output': {}}
if not Settings.quiet:
print('Generation of dir-structure.json', end=" ")
for root, _, files in walk(self.path):
directories = root.split(self.path)[1]
local_result = result['output']
for file in files:
file_path = root + directory_separator + file
file_hash = Hash(file_path, 'sha512', True).get()
lines_in_list = [line.rstrip('\n') for line in open(file_path)]
formated_content = ''
for line in lines_in_list:
if line != lines_in_list[-1]:
formated_content += line + '@@@'
else:
formated_content += line
local_result = local_result.setdefault(
directories,
{file: {
'sha512': file_hash,
'content': formated_content
}})
Helpers.Dict(result).to_json(self.structure)
if not Settings.quiet:
print(Settings.done)
def referer(cls, extension):
"""
Return the referer for the given extension.
:pram extension: A string, a valid domain extension.
"""
from PyFunceble import Lookup
manual_server = {
'aaa': 'whois.nic.aaa',
'abb': 'whois.nic.abb',
'able': 'whois.nic.able',
'accenture': 'whois.nic.accenture',
'aetna': 'whois.nic.aetna',
'aig': 'whois.nic.aig',
'americanexpress': 'whois.nic.americanexpress',
'amex': 'whois.nic.amex',
'amica': 'whois.nic.amica',
'amsterdam': 'whois.nic.amsterdam',
'analytics': 'whois.nic.analytics',
'aramco': 'whois.nic.aramco',
'athleta': 'whois.nic.athleta',
'audible': 'whois.nic.audible',
'author': 'whois.nic.author',
'aws': 'whois.nic.aws',
'axa': 'whois.nic.axa',
'azure': 'whois.nic.azure',
'baby': 'whois.nic.baby',
'banamex': 'whois.nic.banamex',
'bananarepublic': 'whois.nic.bananarepublic',
'baseball': 'whois.nic.baseball',
'bharti': 'whois.nic.bharti',
'bing': 'whois.nic.bing',
'bloomberg': 'whois.nic.bloomberg',
'bm': 'whois.afilias-srs.net',
'book': 'whois.nic.book',
'booking': 'whois.nic.booking',
'bot': 'whois.nic.bot',
'bz': 'whois.afilias-grs.net',
'buzz': 'whois.nic.buzz',
'call': 'whois.nic.call',
'calvinklein': 'whois.nic.calvinklein',
'caravan': 'whois.nic.caravan',
'cartier': 'whois.nic.cartier',
'cbn': 'whois.nic.cbn',
'cbre': 'whois.nic.cbre',
'cd': 'chois.nic.cd',
'chase': 'whois.nic.chase',
'circle': 'whois.nic.circle',
'cisco': 'whois.nic.cisco',
'citadel': 'whois.nic.citadel',
'citi': 'whois.nic.citi',
'citic': 'whois.nic.citic',
'cm': 'whois.netcom.cm',
'coupon': 'whois.nic.coupon',
'crown': 'whois.nic.crown',
'crs': 'whois.nic.crs',
'fj': 'whois.usp.ac.fj',
'ga': 'whois.my.ga',
'gh': 'whois.nic.gh',
'kw': 'whois.nic.kw',
'lc': 'whois2.afilias-grs.net',
'lk': 'whois.nic.lk',
'nyc': 'whois.nic.nyc',
'ps': 'whois.pnina.ps',
'ren': 'whois.nic.ren',
'rw': 'whois.ricta.org.rw',
'shop': 'whois.nic.shop',
'sl': 'whois.nic.sl',
'stream': 'whois.nic.stream',
'tokyo': 'whois.nic.tokyo',
'uno': 'whois.nic.uno',
'za': 'whois.registry.net.za'
}
if extension in manual_server:
return manual_server[extension]
else:
whois_record = Lookup().whois(Settings.iana_server,
'hello.' + extension, 10)
if whois_record is not None:
regex_referer = r'(refer:)\s+(.*)'
if Helpers.Regex(whois_record,
regex_referer,
return_data=False).match():
return Helpers.Regex(whois_record,
regex_referer,
return_data=True,
group=2).match()
return None
def execute(self):
"""
Execute the installation or production logic.
"""
replacement_production = {
'to_replace': [
'official_status_index', 'official_down_status',
'official_up_status', 'auto_continue', 'command_before_end',
'custom_ip', 'days_between_db_retest', 'debug', 'domain',
'generate_hosts', 'header_printed', 'to_filter', 'less',
'logs', 'plain_list_domain', 'quiet', 'referer',
'seconds_before_http_timeout', 'share_logs',
'show_execution_time', 'show_percentage', 'split_files',
'travis', 'travis_autosave_minutes', 'travis_autosave_commit',
'travis_autosave_final_commit', 'unified_file', 'link_to_repo',
'iana_server', 'current_datetime', 'number_of_tested',
'number_of_up', 'number_of_down', 'number_of_invalid',
'http_code_status', 'http_code', 'cleaned_done', 'current_dir'
]
}
replacement_installation = {
'current_dir': r"current_dir = '%%current_dir%%'"
}
replacement_list = {}
if self.production:
replacement_list = replacement_production
else:
replacement_list = replacement_installation
if self.data_to_install is not None:
replacement_list.update(self.data_to_install)
script = Helpers.File(self.file_to_install).read()
for to_replace in replacement_list:
if to_replace == 'to_replace' or to_replace == 'to_install':
for variable in replacement_list[to_replace]:
if to_replace == 'to_install':
replacement = variable + ' = ' + \
str(replacement_list[to_replace][variable])
else:
replacement = variable + ' = ' + \
self.default_values()[variable]
script = Helpers.Regex(script,
variable + ' = .*',
replace_with=replacement,
occurences=1).replace()
else:
replacement = to_replace + ' = ' + \
self.default_values()[to_replace]
script = Helpers.Regex(script,
replacement_list[to_replace],
replace_with=replacement,
occurences=1).replace()
Helpers.File(self.file_to_install).write(script, True)
if not Settings.quiet:
print(Settings.done)
def referer(cls, extension):
"""
Return the referer for the given extension.
:pram extension: A string, a valid domain extension.
"""
from PyFunceble import Lookup
manual_server = {
'aaa': 'whois.nic.aaa',
'abb': 'whois.nic.abb',
'able': 'whois.nic.able',
'accenture': 'whois.nic.accenture',
'aetna': 'whois.nic.aetna',
'aig': 'whois.nic.aig',
'americanexpress': 'whois.nic.americanexpress',
'amex': 'whois.nic.amex',
'amica': 'whois.nic.amica',
'amsterdam': 'whois.nic.amsterdam',
'analytics': 'whois.nic.analytics',
'aramco': 'whois.nic.aramco',
'athleta': 'whois.nic.athleta',
'audible': 'whois.nic.audible',
'author': 'whois.nic.author',
'bm': 'whois.afilias-srs.net',
'bz': 'whois.afilias-grs.net',
'cd': 'chois.nic.cd',
'cm': 'whois.netcom.cm',
'fj': 'whois.usp.ac.fj',
'ga': 'whois.my.ga',
'lc': 'whois2.afilias-grs.net',
'lk': 'whois.nic.lk',
'nyc': 'whois.nic.nyc',
'ps': 'whois.pnina.ps',
'ren': 'whois.nic.ren',
'rw': 'whois.ricta.org.rw',
'shop': 'whois.nic.shop',
'sl': 'whois.nic.sl',
'stream': 'whois.nic.stream',
'tokyp': 'whois.nic.tokyo',
'uno': 'whois.nic.uno',
'za': 'whois.registry.net.za'
}
if extension in manual_server:
return manual_server[extension]
else:
whois_record = Lookup().whois(Settings.iana_server,
'hello.' + extension, 10)
if whois_record is not None:
regex_referer = r'(refer:)\s+(.*)'
if Helpers.Regex(whois_record,
regex_referer,
return_data=False).match():
return Helpers.Regex(whois_record,
regex_referer,
return_data=True,
group=2).match()
return None