def __unicode__(self):
method = self.__get_method()
message = self.__get_message()
if const_isstring(self.value):
return const_convert_to_unicode(method + " " + self.value) \
+ ", " + message
return const_convert_to_unicode(method + " " + repr(self.value)) \
+ ", " + message
def __unicode__(self):
method = self.__get_method()
message = self.__get_message()
if const_isstring(self.value):
return const_convert_to_unicode(method + " " + self.value) \
+ ", " + message
return const_convert_to_unicode(method + " " + repr(self.value)) \
+ ", " + message
def _std_write(msg, stderr = False):
if not const_isstring(msg):
msg = repr(msg)
obj = sys.stdout
if stderr:
obj = sys.stderr
try:
obj.write(msg)
except UnicodeEncodeError:
msg = msg.encode('utf-8')
if const_is_python3():
obj.buffer.write(msg)
else:
obj.write(msg)
def is_in_group(self, user_id, group):
self.check_connection()
groups = self.get_user_groups(user_id)
if isinstance(group, int):
if group in groups:
return True
elif const_isstring(group):
self.cursor.execute("""
SELECT group_id FROM """ + self.TABLE_PREFIX + """groups
WHERE group_name = %s""", (group,))
data = self.cursor.fetchone()
if not data:
return False
elif data['group_id'] in groups:
return True
return False
def _show_document(self, entropy_client, doc, repository, pkgkey):
title = const_convert_to_unicode(doc[Document.DOCUMENT_TITLE_ID])
if not title:
title = _("No title")
title = darkgreen(title)
ts = doc.document_timestamp()
ts = entropy.tools.convert_unix_time_to_human_time(ts)
entropy_client.output(" %s [%s|%s|%s|%s|%s|%s]" % (
bold("@@"),
bold(str(doc.document_id())),
darkred(str(doc.document_type())),
darkgreen(repository),
purple(pkgkey),
blue(doc[DocumentFactory.DOCUMENT_USERNAME_ID]),
darkgreen(ts),
)
)
entropy_client.output("\t%s: %s" % (
blue(_("Title")),
title,
)
)
if const_isstring(doc.document_data()):
text = doc.document_data()
else:
text = doc.document_data().tostring()
text = const_convert_to_unicode(text)
self._formatted_print(
entropy_client, text,
"\t%s: " % (blue(_("Content")),), "\t")
entropy_client.output("\t%s: %s" % (
blue(_("Keywords")),
doc.document_keywords(),
)
)
url = doc.document_url()
if url is not None:
entropy_client.output("\t%s: %s" % (
blue(_("Download")),
url,
)
)
def is_in_group(self, user_id, group):
self.check_connection()
groups = self.get_user_groups(user_id)
if isinstance(group, int):
if group in groups:
return True
elif const_isstring(group):
self.cursor.execute(
"""
SELECT group_id FROM """ + self.TABLE_PREFIX + """groups
WHERE group_name = %s""", (group, ))
data = self.cursor.fetchone()
if not data:
return False
elif data['group_id'] in groups:
return True
return False
def _get_noticeboard_hash(self, repository_id):
"""
Return noticeboard hash data.
@param repository_id: repository identifier
@type repository_id: string
"""
nb_data = self.get_noticeboard(repository_id)
mystr = ''
for key in ("description", "pubDate", "title", "link", "id",):
if key not in nb_data:
continue
elem = nb_data[key]
if not const_isstring(elem):
continue
mystr += elem
return entropy.tools.md5string(mystr)
def _copy_herustic_support(self, handler, local_path, txc_basedir,
remote_path):
"""
Determine if it's possible to remote copy the package from other
configured repositories to save bandwidth.
This herustic only works with package files, not repository db files.
Thus, it should be only enabled for these kind of uploads.
"""
pkg_download = self.handlers_data.get('download')
if pkg_download is None:
# unsupported, we need at least package "download" metadatum
# to be able to reconstruct a valid remote URI
return None, None
current_repository_id = self.repo
available_repositories = self._entropy.available_repositories()
test_repositories = []
for repository_id, repo_meta in available_repositories.items():
if current_repository_id == repository_id:
# not me
continue
if repository_id == InstalledPackagesRepository.NAME:
# __system__ repository doesn't have anything remotely
# it's a fake repo, skip
continue
# In order to take advantage of remote copy, it is also required
# that current working uri (handler.get_uri()) is also a packages
# mirror of the other repository.
if handler.get_uri() not in repo_meta['pkg_mirrors']:
# no way
continue
test_repositories.append(repository_id)
if not test_repositories:
# sorry!
return None, None
test_repositories.sort()
local_path_filename = os.path.basename(local_path)
local_md5 = None
for repository_id in test_repositories:
repo_txc_basedir = \
self._entropy.complete_remote_package_relative_path(
pkg_download, repository_id)
test_remote_path = repo_txc_basedir + "/" + local_path_filename
if not handler.is_file(test_remote_path):
# not found on this packages mirror
continue
# then check md5 and compare
remote_md5 = handler.get_md5(test_remote_path)
if not const_isstring(remote_md5):
# transceiver or remote server doesn't support md5sum()
# so cannot verify the integrity
continue
if local_md5 is None:
local_md5 = md5sum(local_path)
if local_md5 == remote_md5:
# yay! we can copy over!
return handler.copy, (test_remote_path, remote_path)
return None, None
def _generic_post_handler(self, function_name, params, file_params,
timeout):
"""
Given a function name and the request data (dict format), do the actual
HTTP request and return the response object to caller.
WARNING: params and file_params dict keys must be ASCII string only.
@param function_name: name of the function that called this method
@type function_name: string
@param params: POST parameters
@type params: dict
@param file_params: mapping composed by file names as key and tuple
composed by (file_name, file object) as values
@type file_params: dict
@param timeout: socket timeout
@type timeout: float
@return: tuple composed by the server response string or None
(in case of empty response) and the HTTPResponse object (useful
for checking response status)
@rtype: tuple
"""
if timeout is None:
timeout = self._default_timeout_secs
multipart_boundary = "---entropy.services,boundary---"
request_path = self._request_path.rstrip("/") + "/" + function_name
const_debug_write(__name__,
"WebService _generic_post_handler, calling: %s at %s -- %s,"
" tx_callback: %s, timeout: %s" % (self._request_host, request_path,
params, self._transfer_callback, timeout,))
connection = None
try:
if self._request_protocol == "http":
connection = httplib.HTTPConnection(self._request_host,
timeout = timeout)
elif self._request_protocol == "https":
connection = httplib.HTTPSConnection(self._request_host,
timeout = timeout)
else:
raise WebService.RequestError("invalid request protocol",
method = function_name)
headers = {
"Accept": "text/plain",
"User-Agent": self._generate_user_agent(function_name),
}
if file_params is None:
file_params = {}
# autodetect file parameters in params
for k in list(params.keys()):
if isinstance(params[k], (tuple, list)) \
and (len(params[k]) == 2):
f_name, f_obj = params[k]
if isinstance(f_obj, file):
file_params[k] = params[k]
del params[k]
elif const_isunicode(params[k]):
# convert to raw string
params[k] = const_convert_to_rawstring(params[k],
from_enctype = "utf-8")
elif not const_isstring(params[k]):
# invalid ?
if params[k] is None:
# will be converted to ""
continue
int_types = const_get_int()
supported_types = (float, list, tuple) + int_types
if not isinstance(params[k], supported_types):
raise WebService.UnsupportedParameters(
"%s is unsupported type %s" % (k, type(params[k])))
list_types = (list, tuple)
if isinstance(params[k], list_types):
# not supporting nested lists
non_str = [x for x in params[k] if not \
const_isstring(x)]
if non_str:
raise WebService.UnsupportedParameters(
"%s is unsupported type %s" % (k,
type(params[k])))
body = None
if not file_params:
headers["Content-Type"] = "application/x-www-form-urlencoded"
encoded_params = urllib_parse.urlencode(params)
data_size = len(encoded_params)
if self._transfer_callback is not None:
self._transfer_callback(0, data_size, False)
if data_size < 65536:
try:
connection.request("POST", request_path, encoded_params,
headers)
except socket.error as err:
raise WebService.RequestError(err,
method = function_name)
else:
try:
connection.request("POST", request_path, None, headers)
except socket.error as err:
raise WebService.RequestError(err,
method = function_name)
sio = StringIO(encoded_params)
data_size = len(encoded_params)
while True:
chunk = sio.read(65535)
if not chunk:
break
try:
connection.send(chunk)
except socket.error as err:
raise WebService.RequestError(err,
method = function_name)
if self._transfer_callback is not None:
self._transfer_callback(sio.tell(),
data_size, False)
# for both ways, send a signal through the callback
if self._transfer_callback is not None:
self._transfer_callback(data_size, data_size, False)
else:
headers["Content-Type"] = "multipart/form-data; boundary=" + \
multipart_boundary
body_file, body_fpath = self._encode_multipart_form(params,
file_params, multipart_boundary)
try:
data_size = body_file.tell()
headers["Content-Length"] = str(data_size)
body_file.seek(0)
if self._transfer_callback is not None:
self._transfer_callback(0, data_size, False)
try:
connection.request("POST", request_path, None, headers)
except socket.error as err:
raise WebService.RequestError(err,
method = function_name)
while True:
chunk = body_file.read(65535)
if not chunk:
break
try:
connection.send(chunk)
except socket.error as err:
raise WebService.RequestError(err,
method = function_name)
if self._transfer_callback is not None:
self._transfer_callback(body_file.tell(),
data_size, False)
if self._transfer_callback is not None:
self._transfer_callback(data_size, data_size, False)
finally:
body_file.close()
os.remove(body_fpath)
try:
response = connection.getresponse()
except socket.error as err:
raise WebService.RequestError(err,
method = function_name)
const_debug_write(__name__, "WebService.%s(%s), "
"response header: %s" % (
function_name, params, response.getheaders(),))
total_length = response.getheader("Content-Length", "-1")
try:
total_length = int(total_length)
except ValueError:
total_length = -1
outcome = const_convert_to_rawstring("")
current_len = 0
if self._transfer_callback is not None:
self._transfer_callback(current_len, total_length, True)
while True:
try:
chunk = response.read(65536)
except socket.error as err:
raise WebService.RequestError(err,
method = function_name)
if not chunk:
break
outcome += chunk
current_len += len(chunk)
if self._transfer_callback is not None:
self._transfer_callback(current_len, total_length, True)
if self._transfer_callback is not None:
self._transfer_callback(total_length, total_length, True)
if const_is_python3():
outcome = const_convert_to_unicode(outcome)
if not outcome:
return None, response
return outcome, response
except httplib.HTTPException as err:
raise WebService.RequestError(err,
method = function_name)
finally:
if connection is not None:
connection.close()
def __str__(self):
method = self.__get_method()
message = self.__get_message()
if const_isstring(self.value):
return method + " " + self.value + ", " + message
return method + " " + repr(self.value) + ", " + message
def _copy_herustic_support(self, handler, local_path,
txc_basedir, remote_path):
"""
Determine if it's possible to remote copy the package from other
configured repositories to save bandwidth.
This herustic only works with package files, not repository db files.
Thus, it should be only enabled for these kind of uploads.
"""
pkg_download = self.handlers_data.get('download')
if pkg_download is None:
# unsupported, we need at least package "download" metadatum
# to be able to reconstruct a valid remote URI
return None, None
current_repository_id = self.repo
available_repositories = self._entropy.available_repositories()
test_repositories = []
for repository_id, repo_meta in available_repositories.items():
if current_repository_id == repository_id:
# not me
continue
if repository_id == InstalledPackagesRepository.NAME:
# __system__ repository doesn't have anything remotely
# it's a fake repo, skip
continue
# In order to take advantage of remote copy, it is also required
# that current working uri (handler.get_uri()) is also a packages
# mirror of the other repository.
if handler.get_uri() not in repo_meta['pkg_mirrors']:
# no way
continue
test_repositories.append(repository_id)
if not test_repositories:
# sorry!
return None, None
test_repositories.sort()
local_path_filename = os.path.basename(local_path)
local_md5 = None
for repository_id in test_repositories:
repo_txc_basedir = \
self._entropy.complete_remote_package_relative_path(
pkg_download, repository_id)
test_remote_path = repo_txc_basedir + "/" + local_path_filename
if not handler.is_file(test_remote_path):
# not found on this packages mirror
continue
# then check md5 and compare
remote_md5 = handler.get_md5(test_remote_path)
if not const_isstring(remote_md5):
# transceiver or remote server doesn't support md5sum()
# so cannot verify the integrity
continue
if local_md5 is None:
local_md5 = md5sum(local_path)
if local_md5 == remote_md5:
# yay! we can copy over!
return handler.copy, (test_remote_path, remote_path)
return None, None
def _generic_post_handler(self, function_name, params, file_params,
timeout):
"""
Given a function name and the request data (dict format), do the actual
HTTP request and return the response object to caller.
WARNING: params and file_params dict keys must be ASCII string only.
@param function_name: name of the function that called this method
@type function_name: string
@param params: POST parameters
@type params: dict
@param file_params: mapping composed by file names as key and tuple
composed by (file_name, file object) as values
@type file_params: dict
@param timeout: socket timeout
@type timeout: float
@return: tuple composed by the server response string or None
(in case of empty response) and the HTTPResponse object (useful
for checking response status)
@rtype: tuple
"""
if timeout is None:
timeout = self._default_timeout_secs
multipart_boundary = "---entropy.services,boundary---"
request_path = self._request_path.rstrip("/") + "/" + function_name
const_debug_write(
__name__,
"WebService _generic_post_handler, calling: %s at %s -- %s,"
" tx_callback: %s, timeout: %s" % (
self._request_host,
request_path,
params,
self._transfer_callback,
timeout,
))
connection = None
try:
if self._request_protocol == "http":
connection = httplib.HTTPConnection(self._request_host,
timeout=timeout)
elif self._request_protocol == "https":
connection = httplib.HTTPSConnection(self._request_host,
timeout=timeout)
else:
raise WebService.RequestError("invalid request protocol",
method=function_name)
headers = {
"Accept": "text/plain",
"User-Agent": self._generate_user_agent(function_name),
}
if file_params is None:
file_params = {}
# autodetect file parameters in params
for k in list(params.keys()):
if isinstance(params[k], (tuple, list)) \
and (len(params[k]) == 2):
f_name, f_obj = params[k]
if isinstance(f_obj, file):
file_params[k] = params[k]
del params[k]
elif const_isunicode(params[k]):
# convert to raw string
params[k] = const_convert_to_rawstring(
params[k], from_enctype="utf-8")
elif not const_isstring(params[k]):
# invalid ?
if params[k] is None:
# will be converted to ""
continue
int_types = const_get_int()
supported_types = (float, list, tuple) + int_types
if not isinstance(params[k], supported_types):
raise WebService.UnsupportedParameters(
"%s is unsupported type %s" % (k, type(params[k])))
list_types = (list, tuple)
if isinstance(params[k], list_types):
# not supporting nested lists
non_str = [x for x in params[k] if not \
const_isstring(x)]
if non_str:
raise WebService.UnsupportedParameters(
"%s is unsupported type %s" %
(k, type(params[k])))
body = None
if not file_params:
headers["Content-Type"] = "application/x-www-form-urlencoded"
encoded_params = urllib_parse.urlencode(params)
data_size = len(encoded_params)
if self._transfer_callback is not None:
self._transfer_callback(0, data_size, False)
if data_size < 65536:
try:
connection.request("POST", request_path,
encoded_params, headers)
except socket.error as err:
raise WebService.RequestError(err,
method=function_name)
else:
try:
connection.request("POST", request_path, None, headers)
except socket.error as err:
raise WebService.RequestError(err,
method=function_name)
sio = StringIO(encoded_params)
data_size = len(encoded_params)
while True:
chunk = sio.read(65535)
if not chunk:
break
try:
connection.send(chunk)
except socket.error as err:
raise WebService.RequestError(err,
method=function_name)
if self._transfer_callback is not None:
self._transfer_callback(sio.tell(), data_size,
False)
# for both ways, send a signal through the callback
if self._transfer_callback is not None:
self._transfer_callback(data_size, data_size, False)
else:
headers["Content-Type"] = "multipart/form-data; boundary=" + \
multipart_boundary
body_file, body_fpath = self._encode_multipart_form(
params, file_params, multipart_boundary)
try:
data_size = body_file.tell()
headers["Content-Length"] = str(data_size)
body_file.seek(0)
if self._transfer_callback is not None:
self._transfer_callback(0, data_size, False)
try:
connection.request("POST", request_path, None, headers)
except socket.error as err:
raise WebService.RequestError(err,
method=function_name)
while True:
chunk = body_file.read(65535)
if not chunk:
break
try:
connection.send(chunk)
except socket.error as err:
raise WebService.RequestError(err,
method=function_name)
if self._transfer_callback is not None:
self._transfer_callback(body_file.tell(),
data_size, False)
if self._transfer_callback is not None:
self._transfer_callback(data_size, data_size, False)
finally:
body_file.close()
os.remove(body_fpath)
try:
response = connection.getresponse()
except socket.error as err:
raise WebService.RequestError(err, method=function_name)
const_debug_write(
__name__, "WebService.%s(%s), "
"response header: %s" % (
function_name,
params,
response.getheaders(),
))
total_length = response.getheader("Content-Length", "-1")
try:
total_length = int(total_length)
except ValueError:
total_length = -1
outcome = const_convert_to_rawstring("")
current_len = 0
if self._transfer_callback is not None:
self._transfer_callback(current_len, total_length, True)
while True:
try:
chunk = response.read(65536)
except socket.error as err:
raise WebService.RequestError(err, method=function_name)
if not chunk:
break
outcome += chunk
current_len += len(chunk)
if self._transfer_callback is not None:
self._transfer_callback(current_len, total_length, True)
if self._transfer_callback is not None:
self._transfer_callback(total_length, total_length, True)
if const_is_python3():
outcome = const_convert_to_unicode(outcome)
if not outcome:
return None, response
return outcome, response
except httplib.HTTPException as err:
raise WebService.RequestError(err, method=function_name)
finally:
if connection is not None:
connection.close()
def __str__(self):
method = self.__get_method()
message = self.__get_message()
if const_isstring(self.value):
return method + " " + self.value + ", " + message
return method + " " + repr(self.value) + ", " + message
def handler_verify_upload(self, local_filepath, uri, counter, maxcount,
tries, remote_md5 = None):
crippled_uri = EntropyTransceiver.get_uri_name(uri)
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
darkgreen(_("verifying upload (if supported)")),
blue(os.path.basename(local_filepath)),
),
importance = 0,
level = "info",
header = red(" @@ "),
back = True
)
valid_remote_md5 = True
# if remote server supports MD5 commands, remote_md5 is filled
if const_isstring(remote_md5):
valid_md5 = is_valid_md5(remote_md5)
ckres = False
if valid_md5: # seems valid
ckres = compare_md5(local_filepath, remote_md5)
if ckres:
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
darkgreen(_("so far, so good!")),
),
importance = 0,
level = "info",
header = red(" @@ ")
)
return True
# ouch!
elif not valid_md5:
# mmmh... malformed md5, try with handlers
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
bold(_("malformed md5 provided to function")),
),
importance = 0,
level = "warning",
header = brown(" @@ ")
)
else: # it's really bad!
self._entropy.output(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
bold(_("remote md5 is invalid")),
),
importance = 0,
level = "warning",
header = brown(" @@ ")
)
valid_remote_md5 = False
return valid_remote_md5 # always valid
def handler_verify_upload(self,
local_filepath,
uri,
counter,
maxcount,
tries,
remote_md5=None):
crippled_uri = EntropyTransceiver.get_uri_name(uri)
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
darkgreen(_("verifying upload (if supported)")),
blue(os.path.basename(local_filepath)),
),
importance=0,
level="info",
header=red(" @@ "),
back=True)
valid_remote_md5 = True
# if remote server supports MD5 commands, remote_md5 is filled
if const_isstring(remote_md5):
valid_md5 = is_valid_md5(remote_md5)
ckres = False
if valid_md5: # seems valid
ckres = compare_md5(local_filepath, remote_md5)
if ckres:
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
darkgreen(_("so far, so good!")),
),
importance=0,
level="info",
header=red(" @@ "))
return True
# ouch!
elif not valid_md5:
# mmmh... malformed md5, try with handlers
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
bold(_("malformed md5 provided to function")),
),
importance=0,
level="warning",
header=brown(" @@ "))
else: # it's really bad!
self._entropy.output("[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
bold(_("remote md5 is invalid")),
),
importance=0,
level="warning",
header=brown(" @@ "))
valid_remote_md5 = False
return valid_remote_md5 # always valid
def __unicode__(self):
if const_isstring(self.value):
return const_convert_to_unicode(self.value)
return const_convert_to_unicode(repr(self.value))
def __str__(self):
if const_isstring(self.value):
return self.value
return repr(self.value)
def __unicode__(self):
if const_isstring(self.value):
return const_convert_to_unicode(self.value)
return const_convert_to_unicode(repr(self.value))
def __str__(self):
if const_isstring(self.value):
return self.value
return repr(self.value)