def execute(self, context):
"""
Execute the bash command in a temporary directory
which will be cleaned afterwards
"""
pc = get_picas_credentials.picas_cred()
self.token_id = context['task_instance'].xcom_pull(
task_ids=self.token_task)['token_type']
tokens = context['task_instance'].xcom_pull(
task_ids=self.token_task)['token_ids']
if self.token_id == None:
raise RuntimeError("Could not get the token list from the " +
str(self.token_task) + " task")
self.initialilze_submitter(location=self.location,
NCPU=self.NCPU,
parameter_step=self.parameter_step)
pc = get_picas_credentials.picas_cred()
logging.info(
"TODO: Just listing all tokens, not just todo tokens: FIX:")
logging.info(self.token_id)
self.submitter.numjobs = int(len(tokens))
if len(tokens) < 1:
raise RuntimeError("Not enough todo tokens to submit!")
with self.submitter as sub:
launch_ID = sub.launch()
return launch_ID
def execute(self, context):
"""
Execute the bash command in a temporary directory
which will be cleaned afterwards
"""
pc = get_picas_credentials.picas_cred()
tok_dict = context['task_instance'].xcom_pull(task_ids=self.token_task)
token_id = tok_dict['token_type']
pc = get_picas_credentials.picas_cred()
th = token.TokenHandler(t_type=token_id,
uname=pc.user,
pwd=pc.password,
dbn=pc.database)
ts = token.Tokenset(th)
ts.add_keys_to_list(key=keyval[0], val=keyval[1])
def check_folder_for_files_from_tokens(task_id, dummy, number, **context):
xcom_results = context['ti'].xcom_pull(task_ids=task_id)
tokens = list(xcom_results['token_ids'])
token_type = xcom_results['token_type']
pc = picas_cred()
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
tokenlist = TokenList(token_type=token_type, database=db)
for token_id in tokens:
tokenlist.append(
caToken(database=db, token_type=token_type, token_id=token_id))
tokenlist.fetch()
for t in tokenlist:
if t['status'] != 'done':
raise RuntimeError(
"Token {} is not in status 'done' but in status {}".format(
t['_id'], t['status']))
print("All jobs in status 'done' ")
locations = [t['Results_location'] for t in tokenlist]
if len(locations) < number:
print("Only {} files found!".format(len(locations)))
for output_file in locations:
c = subprocess.Popen(['uberftp', '-ls', output_file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out = c.communicate()
if not out[0].decode().strip().split('\n')[0]:
print("{} not found".format(output_file))
def test_get_creds_from_env(self):
os.environ['PICAS_USR'] = '******'
os.environ['PICAS_USR_PWD'] = 'picas_test_pass'
os.environ['PICAS_DB'] = 'picas_test_db'
pc = get_pc.picas_cred()
self.assertTrue(pc.user=='picas_test_user')
self.assertTrue(pc.password=='picas_test_pass')
self.assertTrue(pc.database=='picas_test_db')
def archive_all_tokens(token_type, archive_location, delete=False):
pc = picas_cred()
th = Token.Token_Handler(t_type=token_type, uname=pc.user, pwd=pc.password, dbn=pc.database)
token_archive = th.archive_tokens(delete_on_save=delete, compress=True)
logging.info("Archived tokens from " + token_type + " and made an archive: " + token_archive)
logging.info(token_archive + " size is " + str(os.stat(token_archive).st_size))
subprocess.call(['globus-url-copy '+token_archive+" "+archive_location+"/"+token_archive.split('/')[-1]],shell=True)
logging.info("Resulting archive is at "+archive_location+"/"+token_archive.split('/')[-1])
def test_get_cred_from_file(self):
f = tempfile.NamedTemporaryFile(delete=False)
with open(f.name,'w') as _file:
_file.write('user=testuser\n')
_file.write('password=testpasswd\n')
_file.write('database=testdatabase\n')
pc = get_pc.picas_cred(source_file=f.name)
self.assertTrue(pc.user=='testuser')
self.assertTrue(pc.password=='testpasswd')
self.assertTrue(pc.database=='testdatabase')
os.remove(f.name)
def archive_tokens(self):
os.chdir(self.tmpdir)
self.th.archive_tokens(delete_on_save=True)
pc = get_picas_credentials.picas_cred()
now = datetime.datetime.now()
self.tarfile = "tokens_{}_{}_{}_{}-{}-{}.tar.gz".format(
pc.user, pc.database, self.th.t_type, now.year, now.month,
now.date)
with tarfile.open(self.tmpdir + "/" + self.tarfile, "w:gz") as tar:
tar.add(self.tmpdir)
os.chdir(self.oldpwd)
return self.tmpdir + "/" + self.tarfile
def execute(self, context):
self.tmpdir = mkdtemp() #Makes temporary directory
self.oldpwd = os.getcwd()
tok_dict = context['task_instance'].xcom_pull(task_ids=self.t_task)
token_type = tok_dict['token_type']
self.OBSID = tok_dict['OBSID']
pc = get_picas_credentials.picas_cred()
self.th = token.TokenHandler(t_type=token_type,
uname=pc.user,
pwd=pc.password,
dbn=pc.database)
archive_file = self.archive_tokens()
self.upload_archive_to_storage(archive_file)
self.th.purge_tokens()
def get_tokens_from_view(self):
tokens = []
tok_dict = context['task_instance'].xcom_pull(task_ids=self.token_task)
token_id = tok_dict['token_type']
token_view = tok_dict[
'view'] #TODO: Decide if use view here or in initializer
pc = get_picas_credentials.picas_cred()
th = token.TokenHandler(t_type=token_id,
uname=pc.user,
pwd=pc.password,
dbn=pc.database)
for t in th.list_tokens_from_view(token_view):
tokens.append(t)
return tokens
def execute(self, context):
"""
Execute the bash command in a temporary directory
which will be cleaned afterwards
"""
if self.upload_file == None and self.parset_task == None:
raise (Exception("No Parset task nor upload file specified!"))
pc = get_picas_credentials.picas_cred()
if self.pc_database:
pc.database = self.pc_database
tok_dict = context['task_instance'].xcom_pull(task_ids=self.token_task)
token_ids = tok_dict['token_ids']
token_type = tok_dict['token_type']
filename = self.find_upload_file(context)
self.upload(token_ids, token_type, filename)
def get_srmdir_from_token_task(self, token_task_view):
OBSID = token_task_view['OBSID']
t_type = token_task_view['token_type']
view = token_task_view['view']
key = 'RESULTS_DIR'
pc = get_picas_credentials.picas_cred()
th = token.TokenHandler(t_type=t_type,
uname=pc.user,
pwd=pc.password,
dbn=pc.database)
tokens = th.list_tokens_from_view(view)
srmdir = ''
for t in tokens:
if srmdir != '': break
srmdir = str(th.database[t['id']][key]) + str(OBSID)
return srmdir
def upload(self, token_ids, token_type, file_name):
pc = get_picas_credentials.picas_cred()
if self.pc_database:
pc.database = self.pc_database
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
tl = TokenList(token_type=token_type, database=db)
for t_id in token_ids:
tl.append(
caToken(database=db, token_type=token_type, token_id=t_id))
tl.fetch()
tl.add_attachment(file_name, file_name.split('/')[-1])
def check_folder_for_files_from_task(taskid, xcom_key, number, **context):
"""Either uses number to see how many files should be there
or checks the number of tokens in the view TODO:make this the right way"""
xcom_results = context['ti'].xcom_pull(task_ids=taskid)
path = xcom_results[xcom_key]
if isinstance(number, int):
check_folder_for_files(path, number)
else:
if 'view' in xcom_results.keys():
view = xcom_results['view']
if 'token_type' in xcom_results.keys():
t_type = xcom_results['token_type']
pc = picas_cred()
th = token.TokenHandler(t_type=t_type,
uname=pc.user,
pwd=pc.password,
dbn=pc.database)
number = len(th.list_tokens_from_view(view))
check_folder_for_files(path, number)
def get_result_files_from_tokenlist(token_type,
token_ids,
key="Results_location",
**kwargs):
pc = picas_cred()
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
tokenlist = TokenList(token_type=token_type, database=db)
for token_id in token_ids:
tokenlist.append(
caToken(token_id=token_id, token_type=token_type, database=db))
tokenlist.fetch()
results = [i[key] for i in tokenlist if key in i]
if len(results) == 1:
return results[0]
return results
def execute(self, context):
"""
Execute the bash command in a temporary directory
which will be cleaned afterwards
"""
pc = get_picas_credentials.picas_cred()
if self.pc_database:
pc.database = self.pc_database
tok_dict = context['task_instance'].xcom_pull(task_ids=self.token_task)
token_id = tok_dict['token_type']
th = token.TokenHandler(t_type=token_id,
uname=pc.user,
pwd=pc.password,
dbn=pc.database)
for operation, view in self.modification.iteritems():
if operation == 'reset':
th.reset_tokens(view)
if operation == 'delete':
th.delete_tokens(view)
if operation == 'set_to_status': # {'set_to_status':{"view":'view_name',"status":'status'}}
th.set_view_to_status(view['view'], view['status'])
def test_get_creds_from_init(self):
pc = get_pc.picas_cred(usr='******', pwd='tpwd', dbn='tdb')
self.assertTrue(pc.user=='tusr')
self.assertTrue(pc.password=='tpwd')
self.assertTrue(pc.database=='tdb')
def execute(self, context):
"""
Execute the bash command in a temporary directory
which will be cleaned afterwards
"""
srms = self.get_staged_srms(context)
if not srms:
logging.warn("Could not get the list of staged srms!")
logging.info("the list of staged srms is {0}".format(srms))
pc = get_picas_credentials.picas_cred()
if self.pc_database:
pc.database = self.pc_database
if self.fields_task:
task_name = self.fields_task['name']
task_parent_dag = self.fields_task['parent_dag']
try:
app = get_task_instance(
context, task_name,
task_parent_dag)['sanitized_field_name']
except KeyError:
app = get_task_instance(context, task_name,
task_parent_dag)['field_name']
else:
app = srms.obsid
self.t_type = self.t_type + app
tok_settings = json.load(open(self.tok_config, 'rb'))
token_variables = tok_settings['Job']['variables']
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
self.db = client[pc.database]
pipe_type = token_variables['PIPELINE_STEP']
self.token_list = TokenList(database=self.db, token_type=self.t_type)
if self.files_per_token != 1:
d = slice_dicts(
srms.sbn_dict(pref=self.subband_prefix,
suff=self.subband_suffix), self.files_per_token)
else:
d = {}
for i in srms.sbn_dict(pref=self.subband_prefix,
suff=self.subband_suffix):
d[i[0]] = i[1]
for token_file in d:
logging.info("Token file is {}".format(token_file))
with NamedTemporaryFile(delete=False) as savefile:
for line in d[token_file]:
savefile.write("{}\n".format(line).encode('utf-8'))
# pref3_$FIELDNAME_$OBSID_$PIPELINE_SB$SBNUMBER
pipeline_step = pipe_type.split('_')[1]
# logging.info("Pipeline step is {}, type pipe_type is {}.".format(pipe_type, type(pipe_type)))
if 'cal' in pipe_type:
token_id = "{}_{}_{}".format(self.t_type, srms.obsid,
pipeline_step)
elif 'targ' in pipe_type:
token_id = "{}_{}_{}_SB{}".format(self.t_type, srms.obsid,
pipeline_step, token_file)
else:
token_id = "fields_$FIELDNAME_$OBSID_$PIPELINE: {}_{}_{}_{}_{}".format(
self.t_type, token_file, srms.obsid,
pipe_type.split('_')[1], time.time())
logging.info(token_id)
self.token_list.append(
self.build_token(token_id,
attachment={
'name': 'srm.txt',
'location': savefile.name
}))
self.token_list[-1]['STARTSB'] = token_file
os.remove(savefile.name)
self.token_list.add_token_views()
if self.append_task:
logging.info(self.append_task)
logging.info(context)
self.modify_fields(context)
for token in self.token_list:
token["OBSID"] = srms.obsid
token['RESULTS_DIR'] += "/" + str(srms.obsid)
token_variables['OBSID'] = srms.obsid
token_variables['RESULTS_DIR'] += "/" + str(srms.obsid)
# create result directory if not exist
create_gsiftp_directory(token_variables['RESULTS_DIR'])
logging.info('Token type is ' + self.t_type)
logging.info(
'Tokens are available at https://picas-lofar.grid.surfsara.nl:6984/_utils/database.html?'
+ pc.database + '/_design/' + self.t_type +
'/_view/overview_total')
logging.info("Token settings are :")
for i in token_variables.items():
logging.info(str(i))
logging.debug(srms)
self.token_list.save()
results = dict()
results['num_jobs'] = len(d.keys())
results['output_dir'] = token_variables['RESULTS_DIR']
logging.info("output directory is {}".format(results['output_dir']))
results['token_type'] = str(self.t_type)
results['view'] = pipe_type
results['OBSID'] = token_variables['OBSID']
results['token_ids'] = [i['_id'] for i in self.token_list]
return results
from GRID_LRT.Staging import srmlist
from GRID_LRT.token import caToken
from GRID_LRT.token import TokenJsonBuilder
from GRID_LRT.token import TokenList
from GRID_LRT.token import TokenView
print('Using GRID_LRT from ' + GRID_LRT.__file__)
srms = sys.argv[1]
obsid = sys.argv[2]
cal_obsid = sys.argv[3]
tok_type = 'AAA_prefactor_' + obsid
jsonfile = '/project/sksp/Software/lofar-highres-widefield/testdir/test_with_GRID_LRT/lb_step1_sub6asec.json'
print('1) Initializing PiCaS credentials and connect to database.')
pc = picas_cred()
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
print('Connected to database ' + pc.database)
print('2) Creating a list of paths to the files.')
s = srmlist.srmlist()
with open(srms, 'r') as f:
for l in f.readlines():
s.append(l.strip())
print('3) Slicing list into groups.')
g = s.sbn_dict(pref='SB', suff='_')
def test_get_pc_adapter_works(self):
pc = get_pc.picas_cred()
pc_fun = get_pc.get_picas_cred()
self.assertTrue(pc.user==pc_fun.user)
self.assertTrue(pc.password==pc_fun.password)
self.assertTrue(pc.database==pc_fun.database)