def copyone(urlread, urlwrite):
strurlread = str(urlread)
if urlread.isdir():
if args.recursive:
for u in urlread.walkfiles(include=args.include,
exclude=args.exclude,
enterdir=args.enterdir,
skipdir=args.skipdir,
ignorecase=args.ignorecase):
copyone(urlread / u, urlwrite / u)
else:
if args.verbose:
msg = astyle.style_default(
"ucp: ", astyle.style_url(strurlread),
astyle.style_warn(" (directory skipped)"))
stderr.writeln(msg)
else:
if args.verbose:
msg = astyle.style_default("ucp: ",
astyle.style_url(strurlread),
" -> ")
stderr.write(msg)
try:
with contextlib.closing(urlread.open("rb")) as fileread:
with contextlib.closing(urlwrite.open("wb")) as filewrite:
size = 0
while True:
data = fileread.read(262144)
if data:
filewrite.write(data)
size += len(data)
else:
break
if user or group:
urlwrite.chown(user, group)
except Exception as exc:
if args.ignoreerrors:
if args.verbose:
exctype = misc.format_class(exc)
excmsg = str(exc).replace("\n", " ").strip()
msg = astyle.style_error(
f" (failed with {exctype}: {excmsg})")
stderr.writeln(msg)
else:
raise
else:
if args.verbose:
msg = astyle.style_default(astyle.style_url(str(urlwrite)),
f" ({size:,} bytes)")
stderr.writeln(msg)
def copyone(urlread, urlwrite):
strurlread = str(urlread)
if urlread.isdir():
if args.recursive:
for u in urlread.walkfiles(include=args.include, exclude=args.exclude, enterdir=args.enterdir, skipdir=args.skipdir, ignorecase=args.ignorecase):
copyone(urlread/u, urlwrite/u)
else:
if args.verbose:
msg = astyle.style_default("ucp: ", astyle.style_url(strurlread), astyle.style_warn(" (directory skipped)"))
stderr.writeln(msg)
else:
if args.verbose:
msg = astyle.style_default("ucp: ", astyle.style_url(strurlread), " -> ")
stderr.write(msg)
try:
with contextlib.closing(urlread.open("rb")) as fileread:
with contextlib.closing(urlwrite.open("wb")) as filewrite:
size = 0
while True:
data = fileread.read(262144)
if data:
filewrite.write(data)
size += len(data)
else:
break
if user or group:
urlwrite.chown(user, group)
except Exception as exc:
if args.ignoreerrors:
if args.verbose:
exctype = misc.format_class(exc)
excmsg = str(exc).replace("\n", " ").strip()
msg = astyle.style_error(f" (failed with {exctype}: {excmsg})")
stderr.writeln(msg)
else:
raise
else:
if args.verbose:
msg = astyle.style_default(astyle.style_url(str(urlwrite)), f" ({size:,} bytes)")
stderr.writeln(msg)
def close(self):
if self._log:
ul4log = []
jsonlog = []
for (timestamp, tags, tasks, obj) in self._log:
if isinstance(obj, BaseException):
excclass = misc.format_class(obj)
value = str(obj) or None
tb = _formattraceback(obj)
ul4log.append({"type": "exception", "timestamp": timestamp, "class": excclass, "value": value, "traceback": tb, "tasks": tasks})
jsonlog.append({"type": "exception", "timestamp": timestamp.isoformat(), "class": excclass, "value": value, "traceback": tb, "tasks": [task.asjson() for task in tasks]})
else:
message = "\n".join(_formatlines(obj))
ul4log.append({"type": "message", "timestamp": timestamp, "message": message, "tasks": tasks})
jsonlog.append({"type": "message", "timestamp": timestamp.isoformat(), "message": message, "tasks": [task.asjson() for task in tasks]})
jsondata = dict(
projectname=self.job.projectname,
jobname=self.job.jobname,
identifier=self.job.identifier,
log=jsonlog,
countexceptions=self._countexceptions,
countmessages=self._countmessages,
host_name=misc.sysinfo.host_name,
host_fqdn=misc.sysinfo.host_fqdn,
host_ip=misc.sysinfo.host_ip,
host_sysname=misc.sysinfo.host_sysname,
host_nodename=misc.sysinfo.host_nodename,
host_release=misc.sysinfo.host_release,
host_version=misc.sysinfo.host_version,
host_machine=misc.sysinfo.host_machine,
user_name=misc.sysinfo.user_name,
user_uid=misc.sysinfo.user_uid,
user_gid=misc.sysinfo.user_gid,
user_gecos=misc.sysinfo.user_gecos,
user_dir=misc.sysinfo.user_dir,
user_shell=misc.sysinfo.user_shell,
python_executable=misc.sysinfo.python_executable,
python_version=misc.sysinfo.python_version,
pid=misc.sysinfo.pid,
script_name=misc.sysinfo.script_name,
short_script_name=misc.sysinfo.short_script_name,
starttime=self.job.starttime.isoformat() if self.job.starttime else None,
endtime=self.job.endtime.isoformat() if self.job.endtime else None,
logfileurl=self.job.logfileurl,
)
variables = dict(
job=self.job,
sysinfo=misc.sysinfo,
log=ul4log,
countexceptions=self._countexceptions,
countmessages=self._countmessages,
)
emailsubject = self.job._formatemailsubject.renders(**variables)
emailbodytext = self.job._formatemailbodytext.renders(**variables)
emailbodyhtml = self.job._formatemailbodyhtml.renders(**variables)
textpart = text.MIMEText(emailbodytext)
htmlpart = text.MIMEText(emailbodyhtml, _subtype="html")
jsonpart = application.MIMEApplication(json.dumps(jsondata).encode("utf-8"), _subtype="json", _encoder=encoders.encode_base64)
jsonpart.add_header('Content-Disposition', 'attachment', filename=f"{self.job.projectname}.{self.job.jobname}.json")
msg = multipart.MIMEMultipart(
_subparts=[
multipart.MIMEMultipart(_subtype="alternative", _subparts=[textpart, htmlpart]),
jsonpart,
]
)
msg["To"] = self.job.toemail
msg["From"] = self.job.fromemail
msg["Subject"] = emailsubject
try:
server = smtplib.SMTP(self.job.smtphost, self.job.smtpport)
if self.job.smtpuser and self.job.smtppassword:
server.login(self.job.smtpuser, self.job.smtppassword)
server.send_message(msg)
server.quit()
self.job.log.sisyphus.report(f"Sent email report to {self.job.toemail}")
except smtplib.SMTPException as exc:
self.job.log.sisyphus.report(exc)
def test_format_class():
import http.client
assert "ValueError" == misc.format_class(ValueError())
assert "http.client.HTTPException" == misc.format_class(
http.client.HTTPException())