def make_srmlist_from_gsiftpdir(gsiftpdir): from GRID_LRT.storage import gsifile srml = srmlist() grid_dir = gsifile.GSIFile(gsiftpdir) for i in [f.loc for f in grid_dir.list_dir()]: srml.append(i) return srml
def copy_to_archive( src_dir='gsiftp://gridftp.grid.sara.nl:2811/pnfs/grid.sara.nl/data/lofar/user/sksp/distrib/SKSP', dest_dir='gsiftp://gridftp.grid.sara.nl:2811/pnfs/grid.sara.nl/data/lofar/user/sksp/archive/SKSP/', **context): OBSID = get_task_instance(context, 'get_field_properties')['target_OBSID'] src_dir = src_dir + '/' + OBSID + "/" dest_dir = dest_dir + "/" + OBSID + "/" logging.info("Destination directory is {}".format(dest_dir)) mk_res = subprocess.Popen(['uberftp', '-mkdir', dest_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) mk_res.communicate() src_files = gsifile.GSIFile(src_dir).list_dir() dest_dir_obj = gsifile.GSIFile(dest_dir) for source_file in src_files: copy_gsifile(source_file, dest_dir_obj)
def get_cal_from_dir(base_dir, return_key=None, **context): cal_OBSID = get_task_instance(context, 'get_field_properties')['calib_OBSID'] directory = str(base_dir + cal_OBSID) fold = gsifile.GSIFile(directory) calfile = directory + '/CI_pref3_' + cal_OBSID + '.tar' return {return_key: calfile}
def get_list_from_dir(base_dir, **context): OBSID = context['dag_run'].conf.get('OBSID') OBSID = "1231231" directory = str(base_dir + OBSID) fold = gsifile.GSIFile(directory) files = [i.location for i in fold.list_dir()] files = [] return files
def glfd(base_dir, **context): OBSID = context['dag_run'].conf.get('OBSID') directory = str(base_dir + OBSID) logging.info("Listing files in " + directory) fold = gsifile.GSIFile(directory) files = [i.location for i in fold.list_dir()] srm_file = tempfile.NamedTemporaryFile(delete=False) with open(srm_file.name, 'w') as _f: for f in files: if "GSM" in f: _f.write(f + '\n') if len(files) == 0: raise Exception("No files were found!") return {'srmfile': srm_file.name}
def count_files_uberftp(directory): from GRID_LRT.storage import gsifile grid_dir = gsifile.GSIFile(directory) return [f.location for f in grid_dir.list_dir()]