def make_srmlist_from_gsiftpdir(gsiftpdir):
from GRID_LRT.storage import gsifile
srml = srmlist()
grid_dir = gsifile.GSIFile(gsiftpdir)
for i in [f.loc for f in grid_dir.list_dir()]:
srml.append(i)
return srml
def copy_to_archive(
src_dir='gsiftp://gridftp.grid.sara.nl:2811/pnfs/grid.sara.nl/data/lofar/user/sksp/distrib/SKSP',
dest_dir='gsiftp://gridftp.grid.sara.nl:2811/pnfs/grid.sara.nl/data/lofar/user/sksp/archive/SKSP/',
**context):
OBSID = get_task_instance(context, 'get_field_properties')['target_OBSID']
src_dir = src_dir + '/' + OBSID + "/"
dest_dir = dest_dir + "/" + OBSID + "/"
logging.info("Destination directory is {}".format(dest_dir))
mk_res = subprocess.Popen(['uberftp', '-mkdir', dest_dir],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
mk_res.communicate()
src_files = gsifile.GSIFile(src_dir).list_dir()
dest_dir_obj = gsifile.GSIFile(dest_dir)
for source_file in src_files:
copy_gsifile(source_file, dest_dir_obj)
def get_cal_from_dir(base_dir, return_key=None, **context):
cal_OBSID = get_task_instance(context,
'get_field_properties')['calib_OBSID']
directory = str(base_dir + cal_OBSID)
fold = gsifile.GSIFile(directory)
calfile = directory + '/CI_pref3_' + cal_OBSID + '.tar'
return {return_key: calfile}
def get_list_from_dir(base_dir, **context):
OBSID = context['dag_run'].conf.get('OBSID')
OBSID = "1231231"
directory = str(base_dir + OBSID)
fold = gsifile.GSIFile(directory)
files = [i.location for i in fold.list_dir()]
files = []
return files
def glfd(base_dir, **context):
OBSID = context['dag_run'].conf.get('OBSID')
directory = str(base_dir + OBSID)
logging.info("Listing files in " + directory)
fold = gsifile.GSIFile(directory)
files = [i.location for i in fold.list_dir()]
srm_file = tempfile.NamedTemporaryFile(delete=False)
with open(srm_file.name, 'w') as _f:
for f in files:
if "GSM" in f:
_f.write(f + '\n')
if len(files) == 0:
raise Exception("No files were found!")
return {'srmfile': srm_file.name}
def count_files_uberftp(directory):
from GRID_LRT.storage import gsifile
grid_dir = gsifile.GSIFile(directory)
return [f.location for f in grid_dir.list_dir()]