def check_folder_for_files_from_tokens(task_id, dummy, number, **context):
xcom_results = context['ti'].xcom_pull(task_ids=task_id)
tokens = list(xcom_results['token_ids'])
token_type = xcom_results['token_type']
pc = picas_cred()
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
tokenlist = TokenList(token_type=token_type, database=db)
for token_id in tokens:
tokenlist.append(
caToken(database=db, token_type=token_type, token_id=token_id))
tokenlist.fetch()
for t in tokenlist:
if t['status'] != 'done':
raise RuntimeError(
"Token {} is not in status 'done' but in status {}".format(
t['_id'], t['status']))
print("All jobs in status 'done' ")
locations = [t['Results_location'] for t in tokenlist]
if len(locations) < number:
print("Only {} files found!".format(len(locations)))
for output_file in locations:
c = subprocess.Popen(['uberftp', '-ls', output_file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out = c.communicate()
if not out[0].decode().strip().split('\n')[0]:
print("{} not found".format(output_file))
def upload(self, token_ids, token_type, file_name):
pc = get_picas_credentials.picas_cred()
if self.pc_database:
pc.database = self.pc_database
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
tl = TokenList(token_type=token_type, database=db)
for t_id in token_ids:
tl.append(
caToken(database=db, token_type=token_type, token_id=t_id))
tl.fetch()
tl.add_attachment(file_name, file_name.split('/')[-1])
def get_result_files_from_tokenlist(token_type,
token_ids,
key="Results_location",
**kwargs):
pc = picas_cred()
client = CouchDB(pc.user,
pc.password,
url='https://picas-lofar.grid.surfsara.nl:6984',
connect=True)
db = client[pc.database]
tokenlist = TokenList(token_type=token_type, database=db)
for token_id in token_ids:
tokenlist.append(
caToken(token_id=token_id, token_type=token_type, database=db))
tokenlist.fetch()
results = [i[key] for i in tokenlist if key in i]
if len(results) == 1:
return results[0]
return results