async def _async_sys(proc,
*args,
write: STRBYTES = None,
**kwargs) -> Tuple[str, str]:
"""Small async wrapper function for :func:`.call_sys_async` to simplify calling ``git``, including detecting git errors"""
stderr_raise = kwargs.pop('stderr_raise', False)
strip = kwargs.pop('strip', True)
out, err = await call_sys_async(proc, *args, write=write, **kwargs)
out, err = stringify(out, if_none=''), stringify(err, if_none='')
if strip: out, err = out.strip(), err.strip()
if len(err) > 1 and stderr_raise:
raise SysCallError(f'Git returned an error: "{err}"')
return out, err
def get(self,
key: Union[bytes, str],
default: Any = None,
fail: bool = False) -> Any:
key = str(stringify(key))
res = self.mcache.get(key)
if empty(res):
if fail: raise CacheNotFound(f'Cache key "{key}" was not found.')
return default
return pickle.loads(res) if self.use_pickle else res
async def commit(self,
message: Optional[Union[str, bool]],
*args,
repo: Optional[str] = None) -> str:
"""
Calls ``git commit`` with the arguments ``-m "message" [args]``, where each positional argument passed after the message
is passed as a command line argument.
To disable prepending ``-m`` to the arguments, pass ``None`` as the ``message`` and only your specified ``args`` will be
passed along to ``git commit``.
You may also set ``message`` to ``False`` which will call ``git commit --allow-empty-message [args]`` instead of
prepending ``-m``.
**Basic Usage**
Standard commit with a message, plus the argument '-a'::
>>> g = Git()
>>> g.commit("added example.txt", "-a")
Pass ``None`` as the message to disable prepending ``-m`` to the git arguments. For example, the following call commits
the current staged changes, while re-using the commit message/author info/timestamp from the previous
commit ``9e27d7233ac5bc59bc37c0572a401068fbd5e6be``::
>>> g.commit(None, "-C", "9e27d7233ac5bc59bc37c0572a401068fbd5e6be")
Pass ``False`` as the message for an easy way to call ``git commit --allow-empty-message``, plus any additional
arguments you might pass::
>>> g.commit(False, '-a')
:param str|bool message: The git commit message to commit with. Alternatively ``None`` to remove the default ``-m``, or
``False`` as a shortcut for ``--allow-empty-message`` instead of ``-m``.
:param args: Additional CLI arguments to pass to ``git commit``
:param str repo: (as a kwarg only!) An absolute path to a Git repository to run ``git commit`` within. By default, this is
``None`` which results in the repo passed in the constructor ( :attr:`.repo` ) being used.
:return str stdout: The string text printed to stdout by ``git commit`` while running the command.
"""
if message is None:
return await self.git("commit", *args, repo=repo)
if message is False:
return await self.git("commit",
"--allow-empty-message",
*args,
repo=repo)
return await self.git("commit",
"-m",
stringify(message),
*args,
repo=repo)
def remove(self, *key: Union[bytes, str]) -> bool:
removed = 0
for k in key:
k = str(stringify(k))
try:
self.get(k, fail=True)
self.mcache.delete(k)
removed += 1
except CacheNotFound:
pass
return removed == len(key)
def generate_http_request(
url="/", host=None, method="GET", user_agent=settings.DEFAULT_USER_AGENT, extra_data: Union[STRBYTES, List[str]] = None,
body: STRBYTES = None, **kwargs
) -> bytes:
method, url = stringify(method), stringify(url)
http_ver = stringify(kwargs.get('http_ver', '1.0'))
data = f"{method.upper()} {url} HTTP/{http_ver}\n"
if host is not None: data += f"Host: {stringify(host)}\n"
if user_agent is not None: data += f"User-Agent: {stringify(user_agent)}\n"
data = byteify(data)
if extra_data is not None:
if isinstance(extra_data, list):
extra_data = [byteify(x) for x in extra_data]
data += b"\n".join(extra_data)
else:
data += byteify(extra_data)
if not data.endswith(b"\n"): data += b"\n"
if body is not None:
data += byteify(body)
if not data.endswith(b"\n"): data += b"\n"
if not data.endswith(b"\n\n"): data += b"\n"
return data
def load_keyfile(cls, filename: Text, password: Text = None):
"""
Returns an instance of :class:`.KeyManager` using a public/private key loaded from disk, instead of
from string/bytes key data.
Example::
>>> km = KeyManager.load_keyfile('id_rsa')
>>> d = km.encrypt('hello world')
>>> km.decrypt(d)
b'hello world'
:param str|bytes filename: The file location where the key is stored
:param str|bytes password: If the key is encrypted, specify the password to decrypt it
:raises InvalidFormat: When the key could not be identified, is not supported, or is corrupted
:raises FileNotFoundError: The given ``filename`` couldn't be found.
:return KeyManager cls: An instance of :class:`.KeyManager` (or child class) initialised with the key
"""
filename = stringify(filename)
with open(filename, 'r') as fp:
data = fp.read()
return cls(key=data, password=password)
async def git(self,
*args,
repo: str = None,
strip=True,
stderr=False) -> Union[str, Tuple[str, str]]:
"""
Wrapper async method for calling ``git`` executable command against a repo - casts args to :class:`str`.
This is intended for use internally by :class:`._AsyncGit` methods, but is under a non-private method name to allow you to call
Git commands semi-directly if the class methods are getting in the way.
Example::
>>> Git().git('commit', '--amend', '-a', '-m', 'lorem ipsum dolor')
:param AnyStr|int|float|object|Any args: Positional command line arguments to pass to ``git``. Most formats are fine, as long
as they can be casted into a :class:`str`
:param str|None repo: The repository to run the command against. When ``None``, uses the value of :attr:`.repo`
:param bool strip: (Default: ``True``) Whether to run :meth:`str.strip` on the string outputs of the command
:param bool stderr: (Default: ``False``) When ``True``, includes the stderr output in the response like so: ``(stdout, stderr)``
When ``False``, simply returns ``stdout`` directly as a :class:`str`
:return str stdout: (when ``stderr=False``) The text outputted to stdout by the command
:return Tuple[str,str] stdout_err: (when ``stderr=True``) A :class:`tuple` containing ``(stdout: str, stderr: str)``
"""
repo = self._repo(repo)
out, err = await _async_sys("git",
*[stringify(a) for a in args],
cwd=repo,
strip=strip)
return (out, err) if stderr else out
def convert_datetime(d,
if_empty=None,
fail_empty=False,
**kwargs) -> Optional[datetime]:
"""
Convert the object ``d`` into a :class:`datetime.datetime` object.
If ``d`` is a string or bytes, then it will be parsed using :func:`dateutil.parser.parse`
If ``d`` is an int/float/Decimal, then it will be assumed to be a unix epoch timestamp.
**Examples**::
>>> convert_datetime("2019-01-01T00:00:00Z") # ISO date/time
datetime.datetime(2019, 1, 1, 0, 0, tzinfo=tzutc())
>>> convert_datetime("01/JAN/2019 00:00:00.0000") # Human date/time with month name
datetime.datetime(2019, 1, 1, 0, 0, tzinfo=tzutc())
>>> convert_datetime(1546300800) # Unix timestamp as integer
datetime.datetime(2019, 1, 1, 0, 0, tzinfo=tzutc())
>>> convert_datetime(1546300800000) # Unix timestamp (milliseconds) as integer
datetime.datetime(2019, 1, 1, 0, 0, tzinfo=tzutc())
:param d: Object to convert into a datetime
:param if_empty: If ``d`` is empty / None, return this value
:param bool fail_empty: (Def: ``False``) If this is True, then if ``d`` is empty, raises :class:`AttributeError`
:key datetime.tzinfo tzinfo: (Default: :class:`dateutil.tz.tzutc`) If no timezone was detected by the parser,
use this timezone. Set this to ``None`` to disable forcing timezone-aware dates.
:raises AttributeError: When ``d`` is empty and ``fail_empty`` is set to True.
:raises dateutil.parser.ParserError: When ``d`` could not be parsed into a date.
:return datetime converted: The converted :class:`datetime.datetime` object.
"""
from dateutil.tz import tzutc
_tzinfo = kwargs.pop('tzinfo', tzutc())
if isinstance(d, datetime):
if d.tzinfo is None and _tzinfo is not None:
d = d.replace(tzinfo=_tzinfo)
return d
# For datetime.date objects, we first convert them into a string, then we can parse them into a datetime + set tzinfo
if isinstance(d, date):
d = str(d)
d = stringify(d) if isinstance(d, bytes) else d
if isinstance(d, (int, float)):
return convert_unixtime_datetime(d)
if isinstance(d, str):
from dateutil.parser import parse, ParserError
try:
t = parse(d)
if t.tzinfo is None and _tzinfo is not None:
t = t.replace(tzinfo=_tzinfo)
return t
except (ParserError, ValueError) as e:
log.info(
"Failed to parse string. Attempting to parse as unix time")
try:
t = convert_unixtime_datetime(d)
return t
except (BaseException, Exception, ParserError) as _err:
log.warning(
"Failed to parse unix time. Re-raising original parser error. Unixtime error was: %s %s",
type(_err), str(_err))
raise e
except ImportError as e:
msg = "ERROR: Could not import 'parse' from 'dateutil.parser'. Please " \
f"make sure 'python-dateutil' is installed. Exception: {type(e)} - {str(e)}"
log.exception(msg)
raise ImportError(msg)
if empty(d):
if fail_empty:
raise AttributeError(
"Error converting datetime. Parameter 'd' was empty!")
return if_empty
try:
log.debug(
"Passed object is not a supported type. Object type: %s || object repr: %s",
type(d), repr(d))
log.debug("Calling convert_datetime with object casted to string: %s",
str(d))
_d = convert_datetime(str(d), fail_empty=True)
d = _d
except Exception as e:
log.info(
"Converted passed object with str() to try and parse string version, but failed."
)
log.info("Exception thrown from convert_datetime(str(d)) was: %s %s",
type(e), str(e))
d = None # By setting d to None, it will trigger the ValueError code below.
if not isinstance(d, datetime):
raise ValueError(
'Timestamp must be either a datetime object, or an ISO8601 string...'
)
return d
def _clean_strs(ob, **kwargs):
try:
return stringify(ob)
except Exception:
return str(repr(ob))
def set(self,
key: Union[bytes, str],
value: Any,
timeout: Optional[int] = DEFAULT_CACHE_TIMEOUT):
v = pickle.dumps(value) if self.use_pickle else value
return self.mcache.set(str(stringify(key)), v, timeout)
def load_key(cls,
data: Text,
password: bytes = None) -> Tuple[combined_key_types, str]:
"""
Load a private/public key from a string or bytes ``data`` containing the key in some format, such as PEM
or OpenSSH. Use :py:meth:`.load_keyfile` to load a key from a file + auto-instantiate KeyManager with it.
**Example:**
>>> key = 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG3v6guHpI/j7AIl3F/EWMSReX8fH8MOSq1bC3ZuEjjC'
>>> KeyManager.load_key(key)
(<cryptography.hazmat.backends.openssl.ed25519._Ed25519PublicKey object at 0x7fa118289ba8>, 'public')
:param str|bytes data: The public/private key data, as a string or bytes
:param str|bytes password: If your key data is encrypted, pass the password in this argument to decrypt it.
:raises InvalidFormat: When the key could not be identified, is not supported, or is corrupted
:return tuple key_data: A tuple containing an instance of the key, and a string ``public`` or ``private``
Example: ``(<_Ed25519PublicKey object>, 'public')``
"""
data = stringify(data)
password = None if not password else byteify(password)
try:
if data[0:4] == 'ssh-' or data[0:6] == 'ecdsa-':
return load_ssh_public_key(byteify(data),
backend=default_backend()), 'public'
if '-----BEGIN PUBLIC' in data:
return load_pem_public_key(byteify(data),
backend=default_backend()), 'public'
if '-----BEGIN PRIVATE' in data:
return load_pem_private_key(
byteify(data),
password=password,
backend=default_backend()), 'private'
except binascii.Error as e:
raise InvalidFormat(
f'Error while decoding key - possibly corrupted base64 encoding. '
f'Original exception: {type(e)} {str(e)}')
except ValueError as e:
raise InvalidFormat(
f'Error while decoding key - key may be corrupted. '
f'Original exception: {type(e)} {str(e)}')
##
# If we couldn't identify the key as PEM or OpenSSH, then fallback to attempting to load it as DER / PKCS12
##
data = byteify(data)
try: # Try loading the key as a DER key
key, ktype = cls._load_der_key(data=data, password=password)
return key, ktype
except InvalidFormat:
pass
try: # Try loading the key as PKCS12
key, ktype, _, _ = cls._load_pkcs12_key(data=data,
password=password)
return key, ktype
except InvalidFormat:
pass
# Otherwise we have no idea what to do with this key. Time to give up.
raise InvalidFormat('Unknown key format...')