class UserDB(object):
def __init__(self, storage, key, namespace):
self.storage = SecureStorage(storage)
self.key = key
self.logger = Logger()
self.namespace = namespace_prefix + namespace
self.dbnamespace = namespace_prefix + "apps"
self.lognamespace = namespace_prefix + "logs"
self.logger.info("UserDB has been initialized. Use namespace %s" %
self.namespace)
@asynchronous
def exists(self, name):
try:
yield self.storage.read(self.namespace, name)
except Exception as err:
self.logger.error(str(err))
yield False
else:
yield True
@asynchronous
def get(self, name):
yield self.storage.read(self.namespace, name)
@asynchronous
def create(self, info):
user_info = dict()
uid = uuid.uuid4().hex
name = info['name']
password = info['password']
exists = yield self.exists(name)
if exists:
raise Exception("Already exists")
# Store user uid
user_info['uid'] = uid
# Store username
user_info['name'] = name
# Crypt user passwd
h = HMAC.new(uid)
h.update(password)
user_info['hash'] = h.hexdigest()
try:
yield self.storage.write(self.namespace, name, user_info, USER_TAG)
except Exception as err:
self.logger.error(str(err))
yield False
else:
yield True
@asynchronous
def remove(self, name):
try:
self.logger.info("Remove user %s" % name)
yield self.storage.remove(self.namespace, name)
except Exception as err:
self.logger.error(repr(err))
try:
self.logger.info("Remove user %s application info" % name)
yield self.storage.remove(self.dbnamespace, name)
except Exception as err:
self.logger.error(repr(err))
try:
self.logger.info("Remove user %s upload logs" % name)
tags = LOG_TAG + [name]
logkeys = yield self.storage.find(self.lognamespace, tags)
self.logger.debug("Uploadlogs keys %s" % logkeys)
for key in logkeys:
yield self.storage.remove(self.lognamespace, key)
except Exception as err:
self.logger.error(repr(err))
@asynchronous
def login(self, name, password):
user_info = yield self.get(name)
self.logger.info(str(user_info))
h = HMAC.new(user_info['uid'])
h.update(password)
self.logger.error("%s %s" % (h.hexdigest(), user_info['hash']))
if (h.hexdigest() == user_info['hash']):
user_info.pop('uid')
yield user_info
else:
raise Exception("Invalid pair of login/password")
@asynchronous
def users(self):
yield self.storage.find(self.namespace, USER_TAG)
@asynchronous
def user_apps(self, user):
apps = list()
try:
raw_apps = yield self.storage.read(self.dbnamespace, user)
apps = msgpack.unpackb(raw_apps)
except Exception as err:
self.logger.error(repr(err))
finally:
yield apps
@asynchronous
def write_app_info(self, user, name):
def handler(data):
apps = list()
if data is None:
apps = list()
else:
apps = msgpack.unpackb(data)
if name in apps:
self.logger.error("App %s already exists" % name)
return None
apps.append(name)
return msgpack.packb(apps)
reader = partial(self.storage.read, self.dbnamespace, user)
writer = lambda result: self.storage.write(self.dbnamespace, user,
result, USER_TAG)
yield self.quasi_atomic_write(reader, writer, handler)
@asynchronous
def write_buildlog(self, user, key, logdata):
tags = LOG_TAG + [user]
yield self.storage.write(self.lognamespace, key, logdata, tags)
@asynchronous
def read_buildlog(self, key):
yield self.storage.read(self.lognamespace, key)
@asynchronous
def list_buildlog(self, user):
tags = [user] if user else LOG_TAG
yield self.storage.find(self.lognamespace, tags)
@asynchronous
def quasi_atomic_write(self, reader, writer, handler):
while True:
data = None
summ = ""
try:
data = yield reader()
summ = hashlib.md5(data).hexdigest()
except Exception as err:
self.logger.error(repr(err))
result = handler(data)
if result is None:
break
try:
data = yield reader()
except Exception as err:
self.logger.error(repr(err))
if data is None or summ == hashlib.md5(data).hexdigest():
self.logger.info("MD5 is still valid. Do write")
yield writer(result)
break
self.logger.info("MD5 mismatchs. Continue")
class MySqlDG(object):
def __init__(self, **config):
self.logger = Logger()
self.place = None
self.tablename = ''
try:
unix_socket = config.get('MysqlSocket',
"/var/run/mysqld/mysqld.sock")
self.dbname = config.get('local_db_name', 'COMBAINE')
self.user = config.get('user', 'root')
self.password = config.get('password', "")
self.db = MySQLdb.connect(unix_socket=unix_socket,
user=self.user,
passwd=self.password)
self.cursor = self.db.cursor()
self.cursor.execute('CREATE DATABASE IF NOT EXISTS %s' %
self.dbname)
self.db.commit()
self.db.select_db(self.dbname)
except Exception as err:
self.logger.error('Error in init MySQLdb %s' % err)
raise Exception
def putData(self, data, tablename):
try:
tablename = tablename.replace('.',
'_').replace('-',
'_').replace('+', '_')
line = None
fname = '/dev/shm/%s-%i' % ('COMBAINE', random.randint(0, 65535))
with open(fname, 'w') as table_file:
for line in data:
table_file.write('GOPA'.join([str(x)
for _, x in line]) + '\n')
table_file.close()
if not line:
self.logger.info("Data for mysql is missed")
os.remove(table_file.name)
return False
self.logger.debug(
'Data has been written to a temporary file %s, size: %d bytes'
% (table_file.name, os.lstat(table_file.name).st_size))
if not self._preparePlace(line):
self.logger.error(
'Unsupported field types. Look at preparePlace()')
return False
self.cursor.execute('DROP TABLE IF EXISTS %s' % tablename)
query = "CREATE TABLE IF NOT EXISTS `%(tablename)s` %(struct)s ENGINE = MEMORY DATA DIRECTORY='/dev/shm/'" % {
'tablename': tablename,
'struct': self.place
}
self.cursor.execute(query)
self.db.commit()
query = "LOAD DATA INFILE '%(filename)s' INTO TABLE `%(tablename)s` FIELDS TERMINATED BY 'GOPA'" % {
'filename': table_file.name,
'tablename': tablename
}
self.cursor.execute(query)
self.db.commit()
if os.path.isfile(table_file.name):
os.remove(table_file.name)
except Exception as err:
self.logger.error('Error in putData %s' % err)
if os.path.isfile(table_file.name):
os.remove(table_file.name)
return False
else:
self.tablename = tablename
return True
def _preparePlace(self, example):
ftypes = {
types.IntType: "BIGINT",
types.UnicodeType: "VARCHAR(200)",
types.StringType: "VARCHAR(200)",
types.FloatType: "DOUBLE"
}
try:
self.place = '( %s )' % ','.join([
" `%s` %s" % (field_name, ftypes[type(field_type)])
for field_name, field_type in example
])
except Exception as err:
self.logger.error('Error in preparePlace() %s' % err)
self.place = None
return False
else:
return True
def perfomCustomQuery(self, query_string):
self.logger.debug("Execute query: %s" % query_string)
self.cursor.execute(query_string)
_ret = self.cursor.fetchall()
self.db.commit()
return _ret
class MySqlDG(object):
def __init__(self, **config):
self.logger = Logger()
self.place = None
self.tablename = ''
try:
# port = config.get('local_db_port', 3306)
unix_socket = config.get('MysqlSocket',
"/var/run/mysqld/mysqld.sock")
self.dbname = config.get('local_db_name', 'COMBAINE')
self.db = MySQLdb.connect(unix_socket=unix_socket, user='******', )
self.cursor = self.db.cursor()
self.cursor.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbname)
self.db.commit()
self.db.select_db(self.dbname)
except Exception as err:
self.logger.error('Error in init MySQLdb %s' % err)
raise Exception
def putData(self, data, tablename):
try:
tablename = tablename.replace('.', '_').replace('-', '_').replace('+', '_')
line = None
fname = '/dev/shm/%s-%i' % ('COMBAINE', random.randint(0, 65535))
with open(fname, 'w') as table_file:
for line in data:
table_file.write('GOPA'.join([str(x) for x in line.values()]) + '\n')
table_file.close()
if not line:
self.logger.info("Data for mysql is missed")
os.remove(table_file.name)
return False
self.logger.debug('Data written to a temporary file %s, size: %d bytes'
% (table_file.name, os.lstat(table_file.name).st_size))
if not self._preparePlace(line):
self.logger.error('Unsupported field types. Look at preparePlace()')
return False
self.cursor.execute('DROP TABLE IF EXISTS %s' % tablename)
query = "CREATE TABLE IF NOT EXISTS %(tablename)s %(struct)s ENGINE = MEMORY DATA DIRECTORY='/dev/shm/'" % {'tablename': tablename,
'struct': self.place}
self.cursor.execute(query)
self.db.commit()
query = "LOAD DATA INFILE '%(filename)s' INTO TABLE %(tablename)s FIELDS TERMINATED BY 'GOPA'" % {'filename': table_file.name,
'tablename': tablename}
self.cursor.execute(query)
self.db.commit()
if os.path.isfile(table_file.name):
os.remove(table_file.name)
except Exception as err:
self.logger.error('Error in putData %s' % err)
if os.path.isfile(table_file.name):
os.remove(table_file.name)
return False
else:
self.tablename = tablename
return True
def _preparePlace(self, example):
ftypes = {types.IntType: "INT",
types.UnicodeType: "VARCHAR(200)",
types.StringType: "VARCHAR(200)",
types.FloatType: "FLOAT"}
try:
self.place = '( %s )' % ','.join([" %s %s" % (field_name,
ftypes[type(field_type)])
for field_name, field_type in example.items()])
except Exception as err:
self.logger.error('Error in preparePlace() %s' % err)
self.place = None
return False
else:
return True
def perfomCustomQuery(self, query_string):
self.logger.debug("Execute query: %s" % query_string)
self.cursor.execute(query_string)
_ret = self.cursor.fetchall()
self.db.commit()
return _ret
def __del__(self):
if self.db:
self.cursor.close()
self.db.commit()
self.db.close()
class UrlFetcher():
def __init__(self, io_loop):
self.io_loop = io_loop
self.http_client = AsyncHTTPClient()
self.logger = Logger()
@chain.source
def perform_request(self, request, response, method):
try:
constants = request_consts[method]
url = request[constants.URL]
timeout = request[constants.TIMEOUT]
http_request = HTTPRequest(url=url, method=method)
http_request.request_timeout = float(timeout)/1000
if method == 'POST':
http_request.body = request[constants.BODY]
#adds cookies to request
params_num = len(request)
if constants.COOKIES <= params_num - 1:
cookies = request[constants.COOKIES]
if len(cookies) > 0:
list_of_cookies = list('{0}={1}'.format(cookie, value) for cookie, value in cookies.iteritems())
cookies_str = '; '.join(list_of_cookies)
http_request.headers.add('Cookie', cookies_str)
#adds headers to request
if constants.HEADERS <= params_num - 1:
for name, values_list in request[constants.HEADERS].iteritems():
for value in values_list:
http_request.headers.add(name, value)
self.logger.info("Downloading {0}, headers {1}, method {2}".format(url, http_request.headers, method))
http_response = yield self.http_client.fetch(http_request)
response_headers = self._get_headers_from_response(http_response)
response.write((True, http_response.body, http_response.code, response_headers,))
response.close()
self.logger.info("{0} has been successfuly downloaded".format(url))
except HTTPError as e:
self.logger.info("Error ({0}) occured while downloading {1}".format(e.message, url))
if e.response is not None:
http_response = e.response
response_headers = self._get_headers_from_response(http_response)
response.write((False, http_response.body, http_response.code, response_headers,))
else:
response.write((False, '', e.code, {},))
response.close()
except socket.gaierror as e:
self.logger.info("Error ({0}) occured while downloading {1}".format(e.message, url))
response.write((False, '', e.errno, {},))
response.close()
except Exception as e:
self.logger.error("Unhandled error ({0}) occured in perform_request, report about this problem "
"to httpclient service developers. Method is {1}, stacktrace is: {2}".format(
e.message, method, traceback.format_exc()))
response.write((False, '', 0, {},))
response.close()
@chain.source
def on_get_request(self, request, response):
try:
request_data_packed = yield request.read()
request_data = msgpack.unpackb(request_data_packed)
yield self.perform_request(request_data, response, 'GET')
except StopIteration:
pass
except Exception as e:
self.logger.error("Unhandled error ({0}) occured in on_get_request, report about this problem "
"to httpclient service developers. Stacktrace is: {1}".format(e.message, traceback.format_exc()))
response.write((False, '', 0, {},))
response.close()
@chain.source
def on_post_request(self, request, response):
try:
request_data_packed = yield request.read()
request_data = msgpack.unpackb(request_data_packed)
yield self.perform_request(request_data, response, 'POST')
except StopIteration:
pass
except Exception as e:
self.logger.error("Unhandled error ({0}) occured in on_post_request, report about this problem "
"to httpclient service developers. Stacktrace is: {1}".format(e.message, traceback.format_exc()))
response.write((False, '', 0, {},))
response.close()
def _get_headers_from_response(self, http_response):
response_headers = {}
for header_tuple in http_response.headers.items():
name = header_tuple[0]
value = header_tuple[1]
if not name in response_headers:
response_headers[name] = []
response_headers[name].append(value)
return response_headers
class UserDB(object):
def __init__(self, storage, key, namespace):
self.storage = SecureStorage(storage)
self.key = key
self.logger = Logger()
self.namespace = namespace_prefix + namespace
self.dbnamespace = namespace_prefix + "apps"
self.lognamespace = namespace_prefix + "logs"
self.logger.info("UserDB has been initialized. Use namespace %s"
% self.namespace)
@asynchronous
def exists(self, name):
try:
yield self.storage.read(self.namespace, name)
except Exception as err:
self.logger.error(str(err))
yield False
else:
yield True
@asynchronous
def get(self, name):
yield self.storage.read(self.namespace, name)
@asynchronous
def create(self, info):
user_info = dict()
uid = uuid.uuid4().hex
name = info['name']
password = info['password']
exists = yield self.exists(name)
if exists:
raise Exception("Already exists")
# Store user uid
user_info['uid'] = uid
# Store username
user_info['name'] = name
# Crypt user passwd
h = HMAC.new(uid)
h.update(password)
user_info['hash'] = h.hexdigest()
try:
yield self.storage.write(self.namespace, name, user_info, USER_TAG)
except Exception as err:
self.logger.error(str(err))
yield False
else:
yield True
@asynchronous
def remove(self, name):
try:
self.logger.info("Remove user %s" % name)
yield self.storage.remove(self.namespace, name)
except Exception as err:
self.logger.error(repr(err))
try:
self.logger.info("Remove user %s application info" % name)
yield self.storage.remove(self.dbnamespace, name)
except Exception as err:
self.logger.error(repr(err))
try:
self.logger.info("Remove user %s upload logs" % name)
tags = LOG_TAG + [name]
logkeys = yield self.storage.find(self.lognamespace, tags)
self.logger.debug("Uploadlogs keys %s" % logkeys)
for key in logkeys:
yield self.storage.remove(self.lognamespace, key)
except Exception as err:
self.logger.error(repr(err))
@asynchronous
def login(self, name, password):
user_info = yield self.get(name)
self.logger.info(str(user_info))
h = HMAC.new(user_info['uid'])
h.update(password)
self.logger.error("%s %s" % (h.hexdigest(), user_info['hash']))
if (h.hexdigest() == user_info['hash']):
user_info.pop('uid')
yield user_info
else:
raise Exception("Invalid pair of login/password")
@asynchronous
def users(self):
yield self.storage.find(self.namespace, USER_TAG)
@asynchronous
def user_apps(self, user):
apps = list()
try:
raw_apps = yield self.storage.read(self.dbnamespace, user)
apps = msgpack.unpackb(raw_apps)
except Exception as err:
self.logger.error(repr(err))
finally:
yield apps
@asynchronous
def write_app_info(self, user, name):
def handler(data):
apps = list()
if data is None:
apps = list()
else:
apps = msgpack.unpackb(data)
if name in apps:
self.logger.error("App %s already exists" % name)
return None
apps.append(name)
return msgpack.packb(apps)
reader = partial(self.storage.read, self.dbnamespace, user)
writer = lambda result: self.storage.write(self.dbnamespace,
user,
result,
USER_TAG)
yield self.quasi_atomic_write(reader, writer, handler)
@asynchronous
def write_buildlog(self, user, key, logdata):
tags = LOG_TAG + [user]
yield self.storage.write(self.lognamespace, key, logdata, tags)
@asynchronous
def read_buildlog(self, key):
yield self.storage.read(self.lognamespace, key)
@asynchronous
def list_buildlog(self, user):
tags = [user] if user else LOG_TAG
yield self.storage.find(self.lognamespace, tags)
@asynchronous
def quasi_atomic_write(self, reader, writer, handler):
while True:
data = None
summ = ""
try:
data = yield reader()
summ = hashlib.md5(data).hexdigest()
except Exception as err:
self.logger.error(repr(err))
result = handler(data)
if result is None:
break
try:
data = yield reader()
except Exception as err:
self.logger.error(repr(err))
if data is None or summ == hashlib.md5(data).hexdigest():
self.logger.info("MD5 is still valid. Do write")
yield writer(result)
break
self.logger.info("MD5 mismatchs. Continue")
