def watch_convert(beamline, year, visit, folder):
mib_dict = check_differences(beamline, year, visit, folder)
# Holder for raw data path
if folder:
raw_location = os.path.join('/dls', beamline, 'data', year, visit,
os.path.relpath(folder))
else:
raw_location = os.path.join('/dls', beamline, 'data', year, visit,
'Merlin')
to_convert = mib_dict['MIB_to_convert']
if bool(to_convert):
convert(beamline, year, visit, to_convert, folder)
else:
watch_check = 'Y'
if (watch_check == 'Y' or watch_check == 'y'):
print(raw_location)
path_to_watch = raw_location
# mib_dict = check_differences(beamline, year, visit, folder)
before = dict([(f, None) for f in os.listdir(path_to_watch)])
while True:
time.sleep(60)
after = dict([(f, None) for f in os.listdir(path_to_watch)])
added = [f for f in after if not f in before]
if added:
print("Added dataset: ", ", ".join(added))
new_data_folder = os.listdir(
path=os.path.join(path_to_watch, added[-1]))
for f in new_data_folder:
if f.endswith('mib'):
wait_flag = 1
while wait_flag == 1:
try:
print('file name: ', f)
# above give the file size from source
# but this below throws an error while copy is not complete:
f_size = os.path.getsize(f)
# print(f_size)
print('file size: ', f_size)
wait_flag = 0
except FileNotFoundError:
time.sleep(20)
print('waiting for mib data to copy!!')
print(
os.path.isfile(
os.path.join(
path_to_watch, added[-1], f)))
if os.path.isfile(
os.path.join(
path_to_watch, added[-1], f)):
wait_flag = 0
else:
pass
mib_dict = check_differences(
beamline, year, visit, folder)
to_convert = mib_dict['MIB_to_convert']
convert(beamline, year, visit, to_convert, folder)
before = after
def main(beamline, year, visit, folder = None):
[to_convert, mib_files] = check_differences(beamline, year, visit, folder)
if bool(to_convert):
convert(beamline, year, visit, mib_files, folder)
else:
print('Nothing to convert here!')
def testName(self):
[to_convert_list,
mib_files_list] = check_differences('e02', '2018', 'em20527-1')
if bool(to_convert_list):
convert('e02', '2018', 'em20527-1', mib_files_list)
else:
print('Nothing to convert here!')
def watch_convert(beamline, year, visit, folder):
mib_dict = check_differences(beamline, year, visit, folder)
# Holder for raw data path
# if folder:
# raw_location = os.path.join('/dls',beamline,'data', year, visit, os.path.relpath(folder))
# else:
# raw_location = os.path.join('/dls',beamline,'data', year, visit, 'Merlin')
to_convert = mib_dict['MIB_to_convert']
if bool(to_convert):
convert(beamline, year, visit, to_convert, folder)
def main(beamline, year, visit, folder, folder_num):
print(beamline, year, visit, folder)
if folder == 'False':
folder = ''
HDF5_dict = check_differences(beamline, year, visit)
else:
HDF5_dict = check_differences(beamline, year, visit, folder)
# proc_path = HDF5_dict['processing_path']
to_convert = HDF5_dict['MIB_to_convert']
folder = to_convert[int(folder_num) -
1].rpartition('/')[0].rpartition(visit)[2][1:]
try:
save_location = os.path.join('/dls', beamline, 'data', year, visit,
'processing', folder)
if os.path.exists(save_location) == False:
os.makedirs(save_location)
watch_convert(beamline, year, visit, folder)
except Exception as e:
print('** ERROR processing** \n ', e)
parser.add_argument('folder_num', help='passed by scheduler')
parser.add_argument('-folder',
default=None,
help='option to add a specific folder within a visit \
to look for data, e.g. sample1/dataset1/. If None the assumption would be to look in Merlin folder'
)
v_help = "Display all debug log messages"
parser.add_argument("-v",
"--verbose",
help=v_help,
action="store_true",
default=False)
args = parser.parse_args()
HDF5_dict = check_differences(args.beamline, args.year, args.visit,
args.folder)
to_convert = HDF5_dict['MIB_to_convert']
print(to_convert)
try:
if args.folder is not None:
save_location = os.path.join('/dls', args.beamline, 'data',
args.year, args.visit, 'processing',
args.folder)
if os.path.exists(save_location) == False:
os.makedirs(save_location)
print(to_convert[int(args.folder_num) - 1])
print(args.folder)
convert(args.beamline,
args.year,
args.visit, [to_convert[int(args.folder_num) - 1]],
sys.path.append('/dls_sw/e02/scripts/batch_mib_convert')
from IdentifyPotentialConversions import check_differences
#maximum number of jobs to run concurrently
max_c = 10
beamline = sys.argv[1]
year = sys.argv[2]
visit = sys.argv[3]
if len(sys.argv) < 5:
folder = None
else:
folder = sys.argv[4]
mib_data_dict = check_differences(beamline, year, visit, folder)
saving_path = mib_data_dict['processing_path']
n_files = len(mib_data_dict['MIB_to_convert'])
print('number of MIB files to convert to HDF5 : ', n_files)
outputs_dir = os.path.join(saving_path, 'logs')
if os.path.exists(outputs_dir) == False:
os.makedirs(outputs_dir)
with open(outputs_dir + '/file_numbers.txt', 'w') as f:
for i in np.arange(n_files):
f.write(str(i) + ' : ' + mib_data_dict['MIB_to_convert'][i] + '\n')
f.write('\n')
def testName(self):
check_differences('e02', '2018', 'em20498-1')