def check_folder_for_files_from_tokens(task_id, dummy, number, **context):
xcom_results = context['ti'].xcom_pull(task_ids=task_id)
tokens = list(xcom_results['token_ids'])
token_type = xcom_results['token_type']
pc = picas_cred()
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
tokenlist = TokenList(token_type=token_type, database=db)
for token_id in tokens:
tokenlist.append(
caToken(database=db, token_type=token_type, token_id=token_id))
tokenlist.fetch()
for t in tokenlist:
if t['status'] != 'done':
raise RuntimeError(
"Token {} is not in status 'done' but in status {}".format(
t['_id'], t['status']))
print("All jobs in status 'done' ")
locations = [t['Results_location'] for t in tokenlist]
if len(locations) < number:
print("Only {} files found!".format(len(locations)))
for output_file in locations:
c = subprocess.Popen(['uberftp', '-ls', output_file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out = c.communicate()
if not out[0].decode().strip().split('\n')[0]:
print("{} not found".format(output_file))
def build_token(self, token_id, attachment=None):
t1 = caToken(database=self.db,
token_type=self.t_type,
token_id=token_id)
t1.build(TokenJsonBuilder(self.tok_config))
logging.info(self.tok_config)
t1.save()
if attachment:
t1.add_attachment(attachment_name=attachment['name'],
filename=attachment['location'])
return t1
def upload(self, token_ids, token_type, file_name):
pc = get_picas_credentials.picas_cred()
if self.pc_database:
pc.database = self.pc_database
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
tl = TokenList(token_type=token_type, database=db)
for t_id in token_ids:
tl.append(
caToken(database=db, token_type=token_type, token_id=t_id))
tl.fetch()
tl.add_attachment(file_name, file_name.split('/')[-1])
def get_result_files_from_tokenlist(token_type,
token_ids,
key="Results_location",
**kwargs):
pc = picas_cred()
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
tokenlist = TokenList(token_type=token_type, database=db)
for token_id in token_ids:
tokenlist.append(
caToken(token_id=token_id, token_type=token_type, database=db))
tokenlist.fetch()
results = [i[key] for i in tokenlist if key in i]
if len(results) == 1:
return results[0]
return results
with open(srms, 'r') as f:
for l in f.readlines():
s.append(l.strip())
print('3) Slicing list into groups.')
g = s.sbn_dict(pref='SB', suff='_')
d = srmlist.slice_dicts(g, 1)
print(d)
print('4) Building token list.')
tl = TokenList(token_type=tok_type, database=db)
for k, v in d.items():
print(v)
tok = caToken(database=db,
token_type=tok_type,
token_id=tok_type + '_sub6asec_' + str(cal_obsid) + '_SB' +
str(k))
with open('temp_srm.txt', 'w') as f:
f.write('\n'.join(v))
tok.build(TokenJsonBuilder(jsonfile))
tok.save()
tok.add_attachment(attachment_name='srm.txt', filename='temp_srm.txt')
tl.append(tok)
tl.save()
for tok in tl:
tok['OBSID'] = obsid
tok['PIPELINE_STEP'] = 'lb_sub6asec_cal1'
tok['status'] = 'queued'
tok.save()
d[source] = sources
del match, source, sources
print('4) Building token list.')
tl = TokenList(token_type=tok_type, database=db)
tokens = tl.list_view_tokens('step3_selfcal_cals')
token_ids = [token['_id'] for token in tokens]
for k, v in d.items():
match = re.search('S\d{1,4}', v[0])
if not match:
raise ValueError('No sourcename extracted!')
else:
source = match.group(0)
if (tok_type + '_scal_' + source + str(cal_obsid)) not in token_ids:
tok = caToken(database=db,
token_type=tok_type,
token_id=tok_type + '_scal_' + source + str(cal_obsid))
with open('temp_srm_{:s}.txt'.format(source), 'w') as f:
f.write('\n'.join(v))
tok.build(TokenJsonBuilder(jsonfile))
tok.save()
tok.add_attachment(attachment_name='srm.txt',
filename='temp_srm_{:s}.txt'.format(source))
tl.append(tok)
else:
continue
tl.add_attachment(
attachment_name='step3_selfcal_calibrators.parset',
filename=
'/project/sksp/Software/lofar-highres-widefield/testdir/test_with_GRID_LRT/step3_selfcal_calibrators.parset'
)