def run(self):
self.p_creds = PicasCred()
self.p_creds.user = self.user
self.p_creds.password = self.password
self.p_creds.database = self.database
self.p_creds.put_picas_creds_in_env()
super(ExampleActor, self).run()
def main(*args):
if len(args) == 6:
pc = PicasCred(usr=args[2], pwd=args[3], dbn=args[1])
value = get_token_field(args[4], args[5], pc)
else:
pc = PicasCred()
dbn = pc.database
usr = pc.user
passw = pc.password
value = get_token_field(args[-2], args[-1], pc)
print(value)
def main(*arguments):
if isinstance(arguments[0], list):
arguments = arguments[0]
if len(arguments) <= 5:
pc = PicasCred()
token_id = arguments[-3]
fieldname = arguments[-2]
value = arguments[-1]
else:
token_id = arguments[0]
fieldname = arguments[1]
value = arguments[2]
database = arguments[3]
username = arguments[4]
password = arguments[5]
pc = PicasCred(usr=username, pwd=password, dbn=database)
set_token_field(token_id, fieldname, value, pc)
def export_dict_to_env(db, variable_dictionary, token_id, db_name=None):
pc = PicasCred(dbn=db_name)
dbn, un, pwd = pc.database, pc.user, pc.password
config_json = get_token_field(token_id, "config.json", pc)
if "_token_keys" in variable_dictionary.keys():
logging.info("Exporting Variables from _token_keys")
for var in variable_dictionary["_token_keys"]:
export_key_to_env(var, variable_dictionary["_token_keys"], token_id, pc)
if "_attachments" in variable_dictionary.keys():
logging.info("Exporting Attachments")
for att_file in variable_dictionary["_attachments"]:
export_attachment_to_env(
att_file, variable_dictionary["_attachments"][att_file], token_id, db
)
def export_tok_keys(cfgfile="tokvar.json", token=None):
# dbn=os.environ['PICAS_DB']
# un=os.environ['PICAS_USR']
# pwd=os.environ['PICAS_USR_PWD']
pc = PicasCred()
try:
with open(cfgfile, "r") as _f:
tokvar = json.load(_f)
except Exception as e:
set_token_field(token["_id"], "output", -2, pc)
raise Exception("tokvar read error {0}".format(str(e)))
server = couchdb.Server("https://picas-lofar.grid.surfsara.nl:6984")
server.resource.credentials = (pc.user, pc.password)
db = server[pc.database]
export_dict_to_env(db, tokvar, token["_id"])
p_usr, p_pwd, p_db = (
picas_credentials.user,
picas_credentials.password,
picas_credentials.database,
)
server.resource.credentials = (p_usr, p_pwd)
db = server[p_db]
token = db[token_id]
currdate = time.strftime("%d/%m/%Y_%H.%M.%S_")
client = CouchClient(
url="https://picas-lofar.grid.surfsara.nl:6984",
db=p_db,
username=p_usr,
password=p_pwd,
)
if not name:
name = currdate + attachment
else:
name = attachment
with open(attachment, "rb") as att:
client.db.put_attachment(token, att, str(name))
if __name__ == "__main__":
try:
pc = PicasCred()
upload_attachment(sys.argv[5], sys.argv[6], pc)
except:
pc = PicasCred(dbn=sys.argv[1], usr=sys.argv[2], pwd=sys.argv[3])
set_token_field(sys.argv[5], sys.argv[6], pc)
class ExampleActor(RunActor):
def __init__(self, iterator, modifier):
self.RUNDIR = os.getcwd()
self.iterator = iterator
self.modifier = modifier
self.client = iterator.client
def run(self):
self.p_creds = PicasCred()
self.p_creds.user = self.user
self.p_creds.password = self.password
self.p_creds.database = self.database
self.p_creds.put_picas_creds_in_env()
super(ExampleActor, self).run()
def create_sandbox(self, json_payload=None):
if not json_payload:
json_payload = self.config
if "sandbox" not in json_payload.keys():
logging.warn("No sandbox configuration")
logging.warn("json_payload keys are {0}".format(
json_payload.keys()))
return
sbx = sandbox.Sandbox(config_json=json_payload["sandbox"])
sbx.build_sandbox(True) # Not needed
def get_image(self, config=None):
"""get_image: Downloads the image in the cwd
and sets the environment variable $SIMG to point to
the local downloaded image.
:param config:Optional config JSON, if None, we use
the ExampleActor.config member
"""
if not config:
config = self.config
logging.info("getting image from {0}".format(config))
simg_config = parse_json_payload(config)
for key in simg_config.keys():
export_variable(key, simg_config[key])
image_location = parse_singularity_link(simg_config["SIMG"],
simg_config.get("SIMG_COMMIT"))
if os.path.isfile(image_location):
os.environ['SIMG'] = image_location
else:
raise IOError("SIMG does not exist!")
@staticmethod
def get_variables_from_config(config, variables=None):
if not variables:
variables = {}
if "variables" in config.keys():
logging.info("Getting variables from config file")
_vars = config["variables"]
for var in _vars:
logging.debug("Setting Environment variable {0} to {1}".format(
var, _vars[var]))
variables[var] = _vars[var]
else:
logging.warn(
"No Variables found in the token config. Nothing is put in the environment!"
)
return {}
return variables
def download_sandbox(self, token):
if "SBXloc" in token.keys():
location = token["SBXloc"]
else:
return None
sbx = sandbox.Sandbox(location=location)
sbx.download_sandbox()
def upload_logs(self, logs_dir):
os.chdir(logs_dir)
logsout = "logs_out"
logserr = "logs_.err"
if os.path.isfile(logsout):
upload_attachment(
token_id=self.token_id,
attachment=logsout,
picas_credentials=self.p_creds,
)
if os.path.isfile(logserr):
upload_attachment(
token_id=self.token_id,
attachment=logsout,
picas_credentials=self.p_creds,
)
def process_token(self, key, token):
# Print token information
os.environ["TOKEN"] = token["_id"]
self.token_id = token["_id"]
logging.info(
"Working on token {0} from databse {1} as user {2}".format(
self.token_id, self.database, self.user))
self.config = token["config.json"]
self.pc = PicasCred(usr=self.user,
pwd=self.password,
dbn=self.database)
variables = self.get_variables_from_config(self.config)
if "container" in self.config.keys(
) or "singularity" in self.config.keys():
set_token_field(token["_id"], "status", "pulling_container",
self.pc)
self.get_image()
set_token_field(token["_id"], "status", "building_sandbox", self.pc)
p = Process(target=self.create_sandbox)
p.start()
logging.info("Creating Sandbox from config: {0}".format(
self.config["sandbox"]))
p.join()
with open(os.devnull, "w") as FNULL:
subprocess.call(["chmod", "a+x", "master.sh"],
stdout=FNULL,
stderr=subprocess.STDOUT)
export_dict_to_env(self.client.db,
variables,
self.token_id,
db_name=self.database)
logging.info("Working on token: " + token["_id"])
## Read tokvar values from token and write to bash variables if not already exist! Save attachments and export abs filename to variable
set_token_field(token["_id"], "status", "launched", self.pc)
# The launched script is simply master.sh with token and picas authen stored in env vars
# master.sh takes the variables straight from the token.
command = "/usr/bin/time -v ./master.sh 2> logs_.err 1> logs_out"
logging.info("executing " + command)
out = execute(command, shell=True)
logging.info("master.sh exit status is " + str(out))
set_token_field(token["_id"], "output", out[0], self.pc)
if out[0] == 0:
set_token_field(token["_id"], "status", "done", self.pc)
logging.info("Job exited OK")
else:
set_token_field(token["_id"], "status", "error", self.pc)
logging.error("Job exited with status {0}".format(out[0]))
self.upload_logs(self.RUNDIR)
# Just attaches all png files in the working directory to the token
self.find_and_upload_files()
self.find_and_upload_files("*.fits")
self.client.modify_token(
self.modifier.close(self.client.db[self.token_id]))
def find_and_upload_files(self, filepattern="*.png"):
sols_search = subprocess.Popen(["find", ".", "-name", filepattern],
stdout=subprocess.PIPE)
result = sols_search.communicate()[0]
for filename in result.split():
if isinstance(filename, bytes):
filename = filename.decode()
upload_attachment(
token_id=self.token_id,
attachment=filename,
picas_credentials=self.p_creds,
)
os.remove(filename)
def process_token(self, key, token):
# Print token information
os.environ["TOKEN"] = token["_id"]
self.token_id = token["_id"]
logging.info(
"Working on token {0} from databse {1} as user {2}".format(
self.token_id, self.database, self.user))
self.config = token["config.json"]
self.pc = PicasCred(usr=self.user,
pwd=self.password,
dbn=self.database)
variables = self.get_variables_from_config(self.config)
if "container" in self.config.keys(
) or "singularity" in self.config.keys():
set_token_field(token["_id"], "status", "pulling_container",
self.pc)
self.get_image()
set_token_field(token["_id"], "status", "building_sandbox", self.pc)
p = Process(target=self.create_sandbox)
p.start()
logging.info("Creating Sandbox from config: {0}".format(
self.config["sandbox"]))
p.join()
with open(os.devnull, "w") as FNULL:
subprocess.call(["chmod", "a+x", "master.sh"],
stdout=FNULL,
stderr=subprocess.STDOUT)
export_dict_to_env(self.client.db,
variables,
self.token_id,
db_name=self.database)
logging.info("Working on token: " + token["_id"])
## Read tokvar values from token and write to bash variables if not already exist! Save attachments and export abs filename to variable
set_token_field(token["_id"], "status", "launched", self.pc)
# The launched script is simply master.sh with token and picas authen stored in env vars
# master.sh takes the variables straight from the token.
command = "/usr/bin/time -v ./master.sh 2> logs_.err 1> logs_out"
logging.info("executing " + command)
out = execute(command, shell=True)
logging.info("master.sh exit status is " + str(out))
set_token_field(token["_id"], "output", out[0], self.pc)
if out[0] == 0:
set_token_field(token["_id"], "status", "done", self.pc)
logging.info("Job exited OK")
else:
set_token_field(token["_id"], "status", "error", self.pc)
logging.error("Job exited with status {0}".format(out[0]))
self.upload_logs(self.RUNDIR)
# Just attaches all png files in the working directory to the token
self.find_and_upload_files()
self.find_and_upload_files("*.fits")
self.client.modify_token(
self.modifier.close(self.client.db[self.token_id]))
except Exception as e:
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
del exc_info
print("Exception occured")
print(str(e.args))
set_token_field(actor.token_id, "launcher_status", str(e.args),
actor.pc)
finally:
with open("{0}/GRID_PiCaS_Launcher.log".format(
__file__.split("__init__")[0])) as f:
print(f.read())
if __name__ == "__main__":
"""Entry point of the Launcher.
The options are either 1: database, 2:username 3: password and 4: picas_token_type
or 1: picas_token_type. In this case, we get the picas credentials from ~/.picasrc
or from the environment variables"""
if len(sys.argv) > 4:
db = str(sys.argv[1])
username = str(sys.argv[2])
password = str(sys.argv[3])
else:
pc = PicasCred()
db = pc.database
username = pc.user
password = pc.password
main(db=db, username=username, password=password)