def input(self,item):
if isinstance(item, str):
assert not os.path.normpath(item).startswith('..'), "Input files for AzureBatch must be under the current working directory. This input file is not: '{0}'".format(item)
itemnorm = "./"+os.path.normpath(item).replace("\\","/")
blobxfer(r"blobxfer.py --skipskip --storageaccountkey {} --upload {} {} {}".format(self.storage_key,self.storage_account,self.blob_fn,itemnorm),wd=".")
elif hasattr(item,"copyinputs"):
item.copyinputs(self)
def _process_result(self, distributable, inputOutputCopier,output_blobfn, run_dir_rel):
inputOutputCopier.output(distributable) # Copy (update) any output files from the blob
blobxfer(r"blobxfer.py --storageaccountkey {0} --download {1} {2}/{3} . --remoteresource result.p".format(self.storage_key,self.storage_account_name,self.container,output_blobfn), wd=run_dir_rel)
resultp_filename = os.path.join(run_dir_rel, "result.p")
with open(resultp_filename, mode='rb') as f:
result = pickle.load(f)
return result
def output(self, item):
if isinstance(item, str):
itemnorm = "./" + os.path.normpath(item).replace("\\", "/")
blobxfer(
r"blobxfer.py --skipskip --storageaccountkey {} --download {} {} {} --remoteresource {}"
.format(self.storage_key, self.storage_account, self.blob_fn,
".", itemnorm),
wd=".")
elif hasattr(item, "copyoutputs"):
item.copyoutputs(self)
def _update_python_path_function(self):
localpythonpath = os.environ.get("PYTHONPATH") #!!should it be able to work without pythonpath being set (e.g. if there was just one file)? Also, is None really the return or is it an exception.
if localpythonpath == None: raise Exception("Expect local machine to have 'pythonpath' set")
for i, localpathpart in enumerate(localpythonpath.split(';')):
logging.info("Updating code on pythonpath as needed: {0}".format(localpathpart))
blobxfer(r"blobxfer.py --skipskip --delete --storageaccountkey {0} --upload {1} {2}-pp-v{3}-{4} .".format(
self.storage_key, #0
self.storage_account_name, #1
self.container, #2
self.pp_version, #3
i, #4
),
wd=localpathpart)