def main(*args):
"""Runner for web crawler"""
with open(PID_FILE, 'w') as pid_file:
pid_file.write(str(os.getpid()))
#parse cofig file for logging options
section = get_section(CONFIG_PATH, 'logging')
enable = section['enable']
log_file = section['log_file']
if enable == 'no':
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(filename=log_file, level=logging.WARNING)
#parse cofig file for crawler options
section = get_section(CONFIG_PATH, 'crawler')
depth = section['depth']
content_type = section['images']
download_path = section['download']
if args:
url = args[0]
else:
#parse script paramenter
url = get_script_params()
if url and urllib.urlopen(url):
logging.info('Crawler started.')
simple_controller = crawler.SimpleController(url, depth, \
content_type, download_path)
simple_controller.download_all()
logging.info('Crawler finished.')
else:
logging.warning('You start crawler without url.')
#!/usr/bin/env python
# coding: utf-8
# version: 0.1.1
from lxml import html
import click
import requests
from config_parser import get_section
jenkins_conf = get_section("jenkins")
class JenkinsRoleStrategy(object):
def __init__(self, url, username, password):
self.username = username
self.password = password
self.url = url
self.uri_prefix = "/role-strategy/strategy"
self.roleTypes = {"globalRoles": "globalRoles",
"projectRoles": "projectRoles",
"slaveRoles": "slaveRoles"}
self.session = requests.Session()
self.session.auth = (self.username, self.password)
self.__get_jenkins_crumb()
self.uri_role_list = "/getAllRoles"
self.uri_role_add = "/addRole"
def __init__(self):
self.myllokup = TemplateLookup\
(directories=[TEMPALTES_PATH])
section = get_section(CONFIG_PATH, 'database')
self.connection = get_database_connection(section['host'], \
section['user'], section['passwd'], section['database'])
return cookie['role'].value
def redirect_to(self, role=1):
"""Redirect to page"""
view = views.Page()
self.wfile.write(view.index_page(role))
def check_perm(self, role, perm_level):
"""Cheking permission"""
return role > perm_level
class ThreadedHTTPServer(ThreadingMixIn, BaseHTTPServer.HTTPServer):
"""Handle requests in a separate thread."""
pass
if __name__ == '__main__':
#parse config for server params
section = get_section(CONFIG_PATH, 'web')
HOST_NAME = section['address']
PORT_NUMBER = int(section['port'])
server = ThreadedHTTPServer((HOST_NAME, PORT_NUMBER), MyHandler)
print 'Starting server, use <Ctrl-C> to stop'
print time.asctime(), "Server Starts - %s:%s" % (HOST_NAME, PORT_NUMBER)
try:
server.serve_forever()
except KeyboardInterrupt:
server.server_close()
print time.asctime(), "Server Stops - %s:%s" % (HOST_NAME, PORT_NUMBER)
def __init__(self, path):
self.path = path
self.imgs = []
section = get_section(CONFIG_PATH, 'database')
self.conn = dbapi.get_database_connection(section['host'], \
section['user'], section['passwd'], section['database'])
return 'Not updated'
def delete_user(connection, user_id):
"""Delete user from database by id."""
cursor = connection.cursor()
sql = """DELETE FROM users WHERE id = '%s'""" % user_id
if cursor.execute(sql):
cursor.close()
connection.commit()
return True
return False
def create_user(connection, name, passwd, salt, role):
"""Create new user."""
cursor = connection.cursor()
sql = """INSERT INTO users (name, pass, salt, role)
VALUES ('%s', '%s', '%s', '%s')""" \
%(name, passwd, salt, role)
if cursor.execute(sql):
cursor.close()
connection.commit()
return True
return False
if __name__ == '__main__':
section = get_section(CONFIG_PATH, 'database')
conn = get_database_connection(section['host'], \
section['user'], section['passwd'], section['database'])
print update_user_info(conn, 1, 'user', 3)