def read_metadata(self):
try:
self.metadata = json.loads(b''.join(self.cache).decode("UTF-8"))
except:
raise ValueError("Failed to load metadata JSON")
if not isinstance(self.metadata, dict):
raise ValueError("Wrong format of metadata")
for key in REQUIRED_MANIFEST_FIELDS.union({"version"}):
if key not in self.metadata:
raise ValueError("%s not found in metadata" % key)
validate_requires(self.metadata['requires'])
def upload(self):
"""
Upload the workflow bundle to VelesForge. Requires write access.
"""
try:
metadata = self._parse_metadata(self.path)
except Exception as e:
self.exception("Failed to upload %s:", self.path)
self.stop(Failure(e), False)
return
for key in REQUIRED_MANIFEST_FIELDS:
if key not in metadata:
raise ValueError("No \"%s\" in %s" %
(key, root.common.forge.manifest))
requires = metadata["requires"]
validate_requires(requires)
vreqfound = False
for req in requires:
if Requirement.parse(req).project_name == veles.__name__:
vreqfound = True
break
if not vreqfound:
velreq = veles.__name__ + ">=" + veles.__version__
self.warning("No VELES core requirement was specified. "
"Appended %s", velreq)
requires.append(velreq)
metadata["version"] = self.version
name = metadata["name"]
workflow = metadata["workflow"]
config = metadata["configuration"]
extra = metadata.get("files", [])
if "image" in metadata:
extra.append(metadata["image"])
files = sorted({workflow, config,
root.common.forge.manifest}.union(extra))
self.info("Uploading %s...", name)
agent = Agent(reactor)
headers = Headers({b'User-Agent': [b'twisted']})
# We will send the following:
# 4 bytes with length of metadata in JSON format
# metadata in JSON format
# tar.gz package
@implementer(IBodyProducer)
class ForgeBodyProducer(object):
def __init__(self, owner):
self.owner = owner
self.writer = None
self.length = UNKNOWN_LENGTH
self.finished = Deferred()
self.consumer = None
self.memory = 0
self.event = threading.Event()
def startProducing(self, consumer):
metabytes = json.dumps(
metadata, sort_keys=True).encode('UTF-8')
self.owner.debug("Metadata size is %d", len(metabytes))
consumer.write(struct.pack("!I", len(metabytes)))
consumer.write(metabytes)
self.consumer = consumer
self.writer.start()
return self.finished
def pauseProducing(self):
pass
def resumeProducing(self):
pass
def stopProducing(self):
if self.writer.is_alive():
self.writer.join()
def send(self, data):
self.consumer.write(data)
self.memory -= len(data)
self.event.set()
def write(self, data):
self.memory += len(data)
while self.memory > ForgeClient.UPLOAD_PENDING_BUFFERS * \
ForgeClient.UPLOAD_TAR_BUFFER_SIZE:
self.owner.debug("Suspended tar pipeline")
self.event.wait()
self.event.clear()
self.owner.debug("Scheduling send(%d bytes), pending %d bytes",
len(data), self.memory - len(data))
reactor.callFromThread(self.send, data)
def close(self):
self.owner.debug("Closing, %d bytes are pending", self.memory)
reactor.callFromThread(self.finished.callback, None)
body = ForgeBodyProducer(self)
def write_package():
tbs = ForgeClient.UPLOAD_TAR_BUFFER_SIZE
with TarFile.open(mode="w|gz", fileobj=body, bufsize=tbs,
dereference=True) as tar:
for file in files:
self.debug("Sending %s", file)
ti = TarInfo(file)
fp = os.path.join(self.path, file)
ti.size = os.path.getsize(fp)
ti.mode = 0o666
with open(fp, "rb") as fd:
tar.addfile(ti, fileobj=fd)
body.close()
writer = threading.Thread(target=write_package)
body.writer = writer
def finished(response):
self.debug("Response from server: %s", response.code)
if response.code != 200:
message = "Response code is %d" % response.code
self.error(message)
self.stop(Failure(Exception(message)), False)
else:
self.stop()
def failed(failure):
if not hasattr(failure.value, "reasons"):
try:
failure.raiseException()
except:
self.exception("Failed to upload %s:", name)
else:
self.error("Failed to upload %s:\n%s", name,
failure.value.reasons[0].getTraceback())
self.stop(failure, False)
url = self.base + root.common.forge.upload_name + "?token=" + self.id
self.debug("Sending the request to %s", url)
d = agent.request(
b'POST', url.encode('charmap'), headers=headers, bodyProducer=body)
d.addCallback(finished)
d.addErrback(failed)