def execute(self, context): """ Execute the bash command in a temporary directory which will be cleaned afterwards """ pc = get_picas_credentials.picas_cred() self.token_id = context['task_instance'].xcom_pull( task_ids=self.token_task)['token_type'] tokens = context['task_instance'].xcom_pull( task_ids=self.token_task)['token_ids'] if self.token_id == None: raise RuntimeError("Could not get the token list from the " + str(self.token_task) + " task") self.initialilze_submitter(location=self.location, NCPU=self.NCPU, parameter_step=self.parameter_step) pc = get_picas_credentials.picas_cred() logging.info( "TODO: Just listing all tokens, not just todo tokens: FIX:") logging.info(self.token_id) self.submitter.numjobs = int(len(tokens)) if len(tokens) < 1: raise RuntimeError("Not enough todo tokens to submit!") with self.submitter as sub: launch_ID = sub.launch() return launch_ID
def execute(self, context): """ Execute the bash command in a temporary directory which will be cleaned afterwards """ pc = get_picas_credentials.picas_cred() tok_dict = context['task_instance'].xcom_pull(task_ids=self.token_task) token_id = tok_dict['token_type'] pc = get_picas_credentials.picas_cred() th = token.TokenHandler(t_type=token_id, uname=pc.user, pwd=pc.password, dbn=pc.database) ts = token.Tokenset(th) ts.add_keys_to_list(key=keyval[0], val=keyval[1])
def check_folder_for_files_from_tokens(task_id, dummy, number, **context): xcom_results = context['ti'].xcom_pull(task_ids=task_id) tokens = list(xcom_results['token_ids']) token_type = xcom_results['token_type'] pc = picas_cred() client = CouchDB(pc.user, pc.password, url='https://picas-lofar.grid.surfsara.nl:6984', connect=True) db = client[pc.database] tokenlist = TokenList(token_type=token_type, database=db) for token_id in tokens: tokenlist.append( caToken(database=db, token_type=token_type, token_id=token_id)) tokenlist.fetch() for t in tokenlist: if t['status'] != 'done': raise RuntimeError( "Token {} is not in status 'done' but in status {}".format( t['_id'], t['status'])) print("All jobs in status 'done' ") locations = [t['Results_location'] for t in tokenlist] if len(locations) < number: print("Only {} files found!".format(len(locations))) for output_file in locations: c = subprocess.Popen(['uberftp', '-ls', output_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out = c.communicate() if not out[0].decode().strip().split('\n')[0]: print("{} not found".format(output_file))
def test_get_creds_from_env(self): os.environ['PICAS_USR'] = '******' os.environ['PICAS_USR_PWD'] = 'picas_test_pass' os.environ['PICAS_DB'] = 'picas_test_db' pc = get_pc.picas_cred() self.assertTrue(pc.user=='picas_test_user') self.assertTrue(pc.password=='picas_test_pass') self.assertTrue(pc.database=='picas_test_db')
def archive_all_tokens(token_type, archive_location, delete=False): pc = picas_cred() th = Token.Token_Handler(t_type=token_type, uname=pc.user, pwd=pc.password, dbn=pc.database) token_archive = th.archive_tokens(delete_on_save=delete, compress=True) logging.info("Archived tokens from " + token_type + " and made an archive: " + token_archive) logging.info(token_archive + " size is " + str(os.stat(token_archive).st_size)) subprocess.call(['globus-url-copy '+token_archive+" "+archive_location+"/"+token_archive.split('/')[-1]],shell=True) logging.info("Resulting archive is at "+archive_location+"/"+token_archive.split('/')[-1])
def test_get_cred_from_file(self): f = tempfile.NamedTemporaryFile(delete=False) with open(f.name,'w') as _file: _file.write('user=testuser\n') _file.write('password=testpasswd\n') _file.write('database=testdatabase\n') pc = get_pc.picas_cred(source_file=f.name) self.assertTrue(pc.user=='testuser') self.assertTrue(pc.password=='testpasswd') self.assertTrue(pc.database=='testdatabase') os.remove(f.name)
def archive_tokens(self): os.chdir(self.tmpdir) self.th.archive_tokens(delete_on_save=True) pc = get_picas_credentials.picas_cred() now = datetime.datetime.now() self.tarfile = "tokens_{}_{}_{}_{}-{}-{}.tar.gz".format( pc.user, pc.database, self.th.t_type, now.year, now.month, now.date) with tarfile.open(self.tmpdir + "/" + self.tarfile, "w:gz") as tar: tar.add(self.tmpdir) os.chdir(self.oldpwd) return self.tmpdir + "/" + self.tarfile
def execute(self, context): self.tmpdir = mkdtemp() #Makes temporary directory self.oldpwd = os.getcwd() tok_dict = context['task_instance'].xcom_pull(task_ids=self.t_task) token_type = tok_dict['token_type'] self.OBSID = tok_dict['OBSID'] pc = get_picas_credentials.picas_cred() self.th = token.TokenHandler(t_type=token_type, uname=pc.user, pwd=pc.password, dbn=pc.database) archive_file = self.archive_tokens() self.upload_archive_to_storage(archive_file) self.th.purge_tokens()
def get_tokens_from_view(self): tokens = [] tok_dict = context['task_instance'].xcom_pull(task_ids=self.token_task) token_id = tok_dict['token_type'] token_view = tok_dict[ 'view'] #TODO: Decide if use view here or in initializer pc = get_picas_credentials.picas_cred() th = token.TokenHandler(t_type=token_id, uname=pc.user, pwd=pc.password, dbn=pc.database) for t in th.list_tokens_from_view(token_view): tokens.append(t) return tokens
def execute(self, context): """ Execute the bash command in a temporary directory which will be cleaned afterwards """ if self.upload_file == None and self.parset_task == None: raise (Exception("No Parset task nor upload file specified!")) pc = get_picas_credentials.picas_cred() if self.pc_database: pc.database = self.pc_database tok_dict = context['task_instance'].xcom_pull(task_ids=self.token_task) token_ids = tok_dict['token_ids'] token_type = tok_dict['token_type'] filename = self.find_upload_file(context) self.upload(token_ids, token_type, filename)
def get_srmdir_from_token_task(self, token_task_view): OBSID = token_task_view['OBSID'] t_type = token_task_view['token_type'] view = token_task_view['view'] key = 'RESULTS_DIR' pc = get_picas_credentials.picas_cred() th = token.TokenHandler(t_type=t_type, uname=pc.user, pwd=pc.password, dbn=pc.database) tokens = th.list_tokens_from_view(view) srmdir = '' for t in tokens: if srmdir != '': break srmdir = str(th.database[t['id']][key]) + str(OBSID) return srmdir
def upload(self, token_ids, token_type, file_name): pc = get_picas_credentials.picas_cred() if self.pc_database: pc.database = self.pc_database client = CouchDB(pc.user, pc.password, url='https://picas-lofar.grid.surfsara.nl:6984', connect=True) db = client[pc.database] tl = TokenList(token_type=token_type, database=db) for t_id in token_ids: tl.append( caToken(database=db, token_type=token_type, token_id=t_id)) tl.fetch() tl.add_attachment(file_name, file_name.split('/')[-1])
def check_folder_for_files_from_task(taskid, xcom_key, number, **context): """Either uses number to see how many files should be there or checks the number of tokens in the view TODO:make this the right way""" xcom_results = context['ti'].xcom_pull(task_ids=taskid) path = xcom_results[xcom_key] if isinstance(number, int): check_folder_for_files(path, number) else: if 'view' in xcom_results.keys(): view = xcom_results['view'] if 'token_type' in xcom_results.keys(): t_type = xcom_results['token_type'] pc = picas_cred() th = token.TokenHandler(t_type=t_type, uname=pc.user, pwd=pc.password, dbn=pc.database) number = len(th.list_tokens_from_view(view)) check_folder_for_files(path, number)
def get_result_files_from_tokenlist(token_type, token_ids, key="Results_location", **kwargs): pc = picas_cred() client = CouchDB(pc.user, pc.password, url='https://picas-lofar.grid.surfsara.nl:6984', connect=True) db = client[pc.database] tokenlist = TokenList(token_type=token_type, database=db) for token_id in token_ids: tokenlist.append( caToken(token_id=token_id, token_type=token_type, database=db)) tokenlist.fetch() results = [i[key] for i in tokenlist if key in i] if len(results) == 1: return results[0] return results
def execute(self, context): """ Execute the bash command in a temporary directory which will be cleaned afterwards """ pc = get_picas_credentials.picas_cred() if self.pc_database: pc.database = self.pc_database tok_dict = context['task_instance'].xcom_pull(task_ids=self.token_task) token_id = tok_dict['token_type'] th = token.TokenHandler(t_type=token_id, uname=pc.user, pwd=pc.password, dbn=pc.database) for operation, view in self.modification.iteritems(): if operation == 'reset': th.reset_tokens(view) if operation == 'delete': th.delete_tokens(view) if operation == 'set_to_status': # {'set_to_status':{"view":'view_name',"status":'status'}} th.set_view_to_status(view['view'], view['status'])
def test_get_creds_from_init(self): pc = get_pc.picas_cred(usr='******', pwd='tpwd', dbn='tdb') self.assertTrue(pc.user=='tusr') self.assertTrue(pc.password=='tpwd') self.assertTrue(pc.database=='tdb')
def execute(self, context): """ Execute the bash command in a temporary directory which will be cleaned afterwards """ srms = self.get_staged_srms(context) if not srms: logging.warn("Could not get the list of staged srms!") logging.info("the list of staged srms is {0}".format(srms)) pc = get_picas_credentials.picas_cred() if self.pc_database: pc.database = self.pc_database if self.fields_task: task_name = self.fields_task['name'] task_parent_dag = self.fields_task['parent_dag'] try: app = get_task_instance( context, task_name, task_parent_dag)['sanitized_field_name'] except KeyError: app = get_task_instance(context, task_name, task_parent_dag)['field_name'] else: app = srms.obsid self.t_type = self.t_type + app tok_settings = json.load(open(self.tok_config, 'rb')) token_variables = tok_settings['Job']['variables'] client = CouchDB(pc.user, pc.password, url='https://picas-lofar.grid.surfsara.nl:6984', connect=True) self.db = client[pc.database] pipe_type = token_variables['PIPELINE_STEP'] self.token_list = TokenList(database=self.db, token_type=self.t_type) if self.files_per_token != 1: d = slice_dicts( srms.sbn_dict(pref=self.subband_prefix, suff=self.subband_suffix), self.files_per_token) else: d = {} for i in srms.sbn_dict(pref=self.subband_prefix, suff=self.subband_suffix): d[i[0]] = i[1] for token_file in d: logging.info("Token file is {}".format(token_file)) with NamedTemporaryFile(delete=False) as savefile: for line in d[token_file]: savefile.write("{}\n".format(line).encode('utf-8')) # pref3_$FIELDNAME_$OBSID_$PIPELINE_SB$SBNUMBER pipeline_step = pipe_type.split('_')[1] # logging.info("Pipeline step is {}, type pipe_type is {}.".format(pipe_type, type(pipe_type))) if 'cal' in pipe_type: token_id = "{}_{}_{}".format(self.t_type, srms.obsid, pipeline_step) elif 'targ' in pipe_type: token_id = "{}_{}_{}_SB{}".format(self.t_type, srms.obsid, pipeline_step, token_file) else: token_id = "fields_$FIELDNAME_$OBSID_$PIPELINE: {}_{}_{}_{}_{}".format( self.t_type, token_file, srms.obsid, pipe_type.split('_')[1], time.time()) logging.info(token_id) self.token_list.append( self.build_token(token_id, attachment={ 'name': 'srm.txt', 'location': savefile.name })) self.token_list[-1]['STARTSB'] = token_file os.remove(savefile.name) self.token_list.add_token_views() if self.append_task: logging.info(self.append_task) logging.info(context) self.modify_fields(context) for token in self.token_list: token["OBSID"] = srms.obsid token['RESULTS_DIR'] += "/" + str(srms.obsid) token_variables['OBSID'] = srms.obsid token_variables['RESULTS_DIR'] += "/" + str(srms.obsid) # create result directory if not exist create_gsiftp_directory(token_variables['RESULTS_DIR']) logging.info('Token type is ' + self.t_type) logging.info( 'Tokens are available at https://picas-lofar.grid.surfsara.nl:6984/_utils/database.html?' + pc.database + '/_design/' + self.t_type + '/_view/overview_total') logging.info("Token settings are :") for i in token_variables.items(): logging.info(str(i)) logging.debug(srms) self.token_list.save() results = dict() results['num_jobs'] = len(d.keys()) results['output_dir'] = token_variables['RESULTS_DIR'] logging.info("output directory is {}".format(results['output_dir'])) results['token_type'] = str(self.t_type) results['view'] = pipe_type results['OBSID'] = token_variables['OBSID'] results['token_ids'] = [i['_id'] for i in self.token_list] return results
from GRID_LRT.Staging import srmlist from GRID_LRT.token import caToken from GRID_LRT.token import TokenJsonBuilder from GRID_LRT.token import TokenList from GRID_LRT.token import TokenView print('Using GRID_LRT from ' + GRID_LRT.__file__) srms = sys.argv[1] obsid = sys.argv[2] cal_obsid = sys.argv[3] tok_type = 'AAA_prefactor_' + obsid jsonfile = '/project/sksp/Software/lofar-highres-widefield/testdir/test_with_GRID_LRT/lb_step1_sub6asec.json' print('1) Initializing PiCaS credentials and connect to database.') pc = picas_cred() client = CouchDB(pc.user, pc.password, url='https://picas-lofar.grid.surfsara.nl:6984', connect=True) db = client[pc.database] print('Connected to database ' + pc.database) print('2) Creating a list of paths to the files.') s = srmlist.srmlist() with open(srms, 'r') as f: for l in f.readlines(): s.append(l.strip()) print('3) Slicing list into groups.') g = s.sbn_dict(pref='SB', suff='_')
def test_get_pc_adapter_works(self): pc = get_pc.picas_cred() pc_fun = get_pc.get_picas_cred() self.assertTrue(pc.user==pc_fun.user) self.assertTrue(pc.password==pc_fun.password) self.assertTrue(pc.database==pc_fun.database)