def copy_documentation():
documentation_path = posixpath.join(config.RESOURCE_DIR, 'README.md')
release_documentation_path = config.RELEASE_DIR
utility.make_directory(release_documentation_path)
shutil.copy2(documentation_path, release_documentation_path)
def copy_header_files(softdevice_name='s130'):
serialization_header_path = posixpath.join(config.CODE_DIR, 'inc')
include_override_path = posixpath.join(config.CODE_DIR, 'inc_override')
softdevice_header_path = posixpath.join(config.NRF51_SDK_INCLUDE_DIR, 'softdevice',
softdevice_name, 'headers')
header_file_list = []
file_list_append(header_file_list, serialization_header_path, 'sd_rpc.h')
file_list_append(header_file_list, include_override_path, 'nrf_svc.h')
file_list_append(header_file_list, softdevice_header_path, 'ble.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_hci.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_err.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_types.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_ranges.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_l2cap.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_gap.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_gatt.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_gattc.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_gatts.h')
file_list_append(header_file_list, softdevice_header_path, 'nrf_error.h')
release_include_path = posixpath.join(config.RELEASE_DIR, 'driver', 'include')
utility.make_directory(release_include_path)
for header_file_path in header_file_list:
shutil.copy2(header_file_path, release_include_path)
def __init__(self, configuration):
utility.Print('python_info',
'\nCreated instance of MergeHandler class')
# Get campaign name
self.campaigns = configuration.general.campaigns
self.campaigns_info = configuration.campaigns.info
# Force all regardless of already existing files
self.force_all = configuration.general.force_all
# Work with local files or remote
self.work_locally = configuration.general.work_locally
# ------ Paths -------
self.path_main = configuration.paths.main
self.path_logical_file_names = configuration.paths.logical_file_names
self.path_histograms = utility.make_directory(
configuration.paths.histograms)
self.path_samples = configuration.paths.samples
self.path_plots_final = utility.make_directory(
configuration.paths.plots_final)
# ------ Samples -------
self.samples_info = configuration.samples.info
# ------ Browsing options -------
self.remote_locations = configuration.general.remote_locations
# Merge datasets configs
self.luminosity = configuration.general.luminosity
self.number_of_files = configuration.general.number_of_files
self.analyzer_module = configuration.general.analyzer_module
self.groups = configuration.general.groups
def create_image_info(input_file_path, output_file_path):
if os.path.isfile(output_file_path):
return
print("{} -> {}".format(input_file_path, output_file_path))
img = Image.open(input_file_path)
exif_dict = img._getexif()
exif_result_info = {}
for tag_id, value in exif_dict.items():
tag = ExifTags.TAGS.get(tag_id, tag_id)
if isinstance(value, bytes):
continue
exif_result_info[tag] = value
# print(tag_dict)
"""
exif_result_info = {}
for k, v in exif_dict.items():
tag_value = jpeg_tag_dict.get(k, None)
if tag_value:
tag_name = tag_value[0]
exif_result_info[tag_name] = v
"""
utility.make_directory(output_file_path)
write_data = json.dumps(
exif_result_info,
cls=JsonEncoder,
indent=" ",
)
with open(output_file_path, "w") as fp:
fp.write(write_data)
def copy_header_files(softdevice_name='s130'):
serialization_header_path = posixpath.join(config.CODE_DIR, 'inc')
include_override_path = posixpath.join(config.CODE_DIR, 'inc_override')
softdevice_header_path = posixpath.join(config.NRF51_SDK_INCLUDE_DIR,
'softdevice', softdevice_name,
'headers')
header_file_list = []
file_list_append(header_file_list, serialization_header_path, 'sd_rpc.h')
file_list_append(header_file_list, include_override_path, 'nrf_svc.h')
file_list_append(header_file_list, softdevice_header_path, 'ble.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_hci.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_err.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_types.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_ranges.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_l2cap.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_gap.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_gatt.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_gattc.h')
file_list_append(header_file_list, softdevice_header_path, 'ble_gatts.h')
file_list_append(header_file_list, softdevice_header_path, 'nrf_error.h')
release_include_path = posixpath.join(config.RELEASE_DIR, 'driver',
'include')
utility.make_directory(release_include_path)
for header_file_path in header_file_list:
shutil.copy2(header_file_path, release_include_path)
def copy_python_binding_release_files(softdevice_name='s130'):
s130_bindings_release_path = posixpath.join(config.RELEASE_DIR, 'python')
utility.make_clean_directory(s130_bindings_release_path)
python_binding_lib_name = softdevice_name + '_' + config.LIB_NAME
python_binding_py_name = python_binding_lib_name + '.py'
python_binding_pyd_name = '_' + python_binding_lib_name + config.PYTHON_BINDING_SUFFIX
python_bindings_build_path = posixpath.join(config.BUILD_DIR, 'python')
python_binding_release_path = posixpath.join(config.RELEASE_DIR, 'python')
utility.make_directory(python_binding_release_path)
python_binding_py_path = posixpath.join(python_bindings_build_path,
python_binding_py_name)
python_binding_pyd_path = posixpath.join(python_bindings_build_path,
python_binding_pyd_name)
shutil.copy2(python_binding_py_path, python_binding_release_path)
shutil.copy2(python_binding_pyd_path, python_binding_release_path)
python_binding_path = config.BINDINGS_DIR
python_binding_examples_path = posixpath.join(python_binding_path, 'examples')
python_binding_examples_release_path = posixpath.join(python_binding_release_path, 'examples')
python_bindings_util_path = posixpath.join(python_binding_path, 'src', 'ble_driver_util.py')
shutil.copy2(python_bindings_util_path, python_binding_release_path)
shutil.copytree(python_binding_examples_path, python_binding_examples_release_path)
def combine_file(combine_file_path, pic_file_path_0, pic_file_path_1):
def _get_mask_image(target_image_size):
for ref_image_size in ((1200, 800), (1124, 800), (800, 1154),
(800, 1200), (1116, 800), (1187, 800), (1167,
800)):
if abs((target_image_size[0] / target_image_size[1]) -
(ref_image_size[0] / ref_image_size[1])) < 0.02:
mask_image = Image.open(
os.path.join(
settings.MASK_DIRECTORY_PATH,
"mask_{}_{}.bmp".format(*ref_image_size))).convert(
"1").resize(target_image_size)
return mask_image
return None
image_0 = Image.open(pic_file_path_0)
image_1 = Image.open(pic_file_path_1)
mask_image = _get_mask_image(image_0.size)
print(pic_file_path_0, image_0.size)
if (image_0.size == image_1.size == mask_image.size):
out_image = Image.composite(image_1, image_0, mask_image)
print(combine_file_path, out_image.size)
utility.make_directory(combine_file_path)
out_image.save(combine_file_path)
def copy_hex_files(softdevice_name='s130'):
hex_file_list = []
version = {
's110': '7.1.0',
's120': '1.0.1',
's130': '1.0.0'
}[softdevice_name]
hex_resource_path = config.RESOURCE_DIR
hex_file_name = 'connectivity_115k2_with_{0}_{1}.hex'.format(
softdevice_name, version)
file_list_append(hex_file_list, hex_resource_path, hex_file_name)
if not config.PLATFORM_SYSTEM == 'Darwin':
hex_file_name = 'connectivity_1m_with_{0}_{1}.hex'.format(
softdevice_name, version)
file_list_append(hex_file_list, hex_resource_path, hex_file_name)
release_hex_path = posixpath.join(config.RELEASE_DIR, 'firmware')
utility.make_directory(release_hex_path)
for hex_file_path in hex_file_list:
shutil.copy2(hex_file_path, release_hex_path)
def create_movie_small(size, bv, ba, input_file_path, output_file_path):
if os.path.isfile(output_file_path):
return
utility.make_directory(output_file_path)
command = (
cf.FFMPEG_PATH,
"-y",
"-i",
input_file_path,
"-movflags",
"faststart",
"-c:v",
"libx264", # libx265 # libx264 # mpeg2 # libxvid
"-vf",
"scale={}:-1".format(size),
"-b:v",
bv,
"-c:a",
"aac",
# "-acodec aac -strict experimental"
"-b:a",
ba,
output_file_path)
print(command)
def call_back(returncode, stdout_message):
print(returncode, stdout_message)
pass
utility.create_process(command, call_back, 1, 2400)()
def __init__(self, configuration):
utility.Print('python_info', '\nCreated instance of FileTool class')
# ------ Paths -------
self.path_main = configuration.paths.main
self.path_samples = utility.make_directory(configuration.paths.samples)
self.path_logical_file_names = utility.make_directory(
configuration.paths.logical_file_names)
self.path_batch_results = utility.make_directory(
configuration.paths.batch_results)
self.path_batch_templates = utility.make_directory(
configuration.paths.batch_templates)
self.path_proxy = configuration.paths.proxy
# ------ Batch options -------
self.batch_type = configuration.general.batch_type
self.number_of_jobs = configuration.general.number_of_jobs
self.send_jobs = configuration.general.send_jobs
self.batch_templates = configuration.general.batch_templates
# ------ Samples -------
self.samples_info = configuration.samples.info
self.samples = self.samples_info.keys()
# ------ Browsing options -------
self.remote_locations = configuration.general.remote_locations
self.search_keywords = configuration.general.search_keywords
def download_pic(opener, eventno, url, file_path):
print("url-> ", url)
print("file_path-> ", file_path)
if opener is None:
return
req = urllib.request.Request(url)
referer_url = '{}/?action_user_FastViewer=t&eventno={}'.format(
settings.URL_ROOT, eventno)
req.add_header('Referer', referer_url)
req.add_header('User-Agent', settings.HTTP_HEADER_USER_AGENT)
req.add_header('accept-encoding', 'gzip, deflate, br')
req.add_header(
'accept',
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3'
)
req.add_header('cache-control', 'max-age=0')
# req.add_header('cookie', JPEG_COOKIE)
jpg = opener.open(req).read()
utility.make_directory(file_path)
with open(file_path, mode="wb") as f:
f.write(jpg)
def copy_python_binding_release_files(softdevice_name='s130'):
s130_bindings_release_path = posixpath.join(config.RELEASE_DIR, 'python')
utility.make_clean_directory(s130_bindings_release_path)
python_binding_lib_name = softdevice_name + '_' + config.LIB_NAME
python_binding_py_name = python_binding_lib_name + '.py'
python_binding_pyd_name = '_' + python_binding_lib_name + config.PYTHON_BINDING_SUFFIX
python_bindings_build_path = posixpath.join(config.BUILD_DIR, 'python')
python_binding_release_path = posixpath.join(config.RELEASE_DIR, 'python')
utility.make_directory(python_binding_release_path)
python_binding_py_path = posixpath.join(python_bindings_build_path,
python_binding_py_name)
python_binding_pyd_path = posixpath.join(python_bindings_build_path,
python_binding_pyd_name)
shutil.copy2(python_binding_py_path, python_binding_release_path)
shutil.copy2(python_binding_pyd_path, python_binding_release_path)
python_binding_path = config.BINDINGS_DIR
python_binding_examples_path = posixpath.join(python_binding_path,
'examples')
python_binding_examples_release_path = posixpath.join(
python_binding_release_path, 'examples')
python_bindings_util_path = posixpath.join(python_binding_path, 'src',
'ble_driver_util.py')
shutil.copy2(python_bindings_util_path, python_binding_release_path)
shutil.copytree(python_binding_examples_path,
python_binding_examples_release_path)
def copy_documentation():
documentation_path = posixpath.join(config.RESOURCE_DIR, 'README.md')
release_documentation_path = config.RELEASE_DIR
utility.make_directory(release_documentation_path)
shutil.copy2(documentation_path, release_documentation_path)
def copy_serialization_dll(softdevice_name='s130'):
dynamic_library_name = config.DYNAMIC_LIB_PREFIX + softdevice_name + '_' + config.LIB_NAME
dynamic_library_file = dynamic_library_name + config.DYNAMIC_LIB_SUFFIX
dynamic_library_path = posixpath.join(config.SERIALIZATION_BUILD_DIR,
dynamic_library_file)
dynamic_library_dest_path = posixpath.join(config.RELEASE_DIR, "driver", "lib")
utility.make_directory(dynamic_library_dest_path)
shutil.copy2(dynamic_library_path, dynamic_library_dest_path)
def build():
logger.info('Building serialization dll artifacts with CMake')
sdk_info = Sdk(config.ARTIFACTS_ROOT, config.SDK_VERSION)
utility.make_directory(config.SERIALIZATION_BUILD_DIR)
if config.PLATFORM_SYSTEM == 'Windows':
generator = 'MinGW Makefiles'
elif config.PLATFORM_SYSTEM in ['Linux', 'Darwin']:
generator = 'Unix Makefiles'
else:
raise SystemError('Unknown platform. Not able to determine generator.')
cmake_environment = None
# Remove any git/bin path in environment variable PATH
if config.PLATFORM_SYSTEM == 'Windows':
environment_path = os.environ['PATH']
environment_path_list = environment_path.split(';')
environment_path_list = [path for path in environment_path_list if 'Git\\bin' not in path]
environment_path = ';'.join(environment_path_list)
cmake_environment = copy.copy(os.environ)
cmake_environment['PATH'] = environment_path
for artifact in ['driver', 'binding']:
cmake_args = ['cmake', '-G', '{0}'.format(generator),
'-DNRF51_SDK_PATH={0}'.format(sdk_info.path),
'-DSERIALIZATION_VERSION={0}'.format(config.VERSION),
'-DSERIALIZATION_REVISION={0}'.format(config.REVISION),
'-DARTIFACT={0}'.format(artifact),
config.REPO_ROOT_DIR]
logging.debug("Starting to build with command: %s", " ".join(cmake_args))
return_code = None
try:
return_code = subprocess.call(cmake_args,
shell=False,
env=cmake_environment)
if return_code != 0:
err_msg = 'Failed to prepare build of {0} libraries. Error code: {1}.'.format(artifact, return_code)
utility.add_log_message(err_msg)
raise SystemError(err_msg)
return_code = subprocess.call([config.PLATFORM_MAKE], shell=True)
if return_code != 0:
err_msg = 'Failed to build artifact {0}. Error code: {0}.'.format(artifact, return_code)
utility.add_log_message(err_msg)
raise SystemError(err_msg)
except Exception, e:
logger.fatal(e)
return return_code
def copy_serialization_dll(softdevice_name='s130'):
dynamic_library_name = config.DYNAMIC_LIB_PREFIX + softdevice_name + '_' + config.LIB_NAME
dynamic_library_file = dynamic_library_name + config.DYNAMIC_LIB_SUFFIX
dynamic_library_path = posixpath.join(config.SERIALIZATION_BUILD_DIR,
dynamic_library_file)
dynamic_library_dest_path = posixpath.join(config.RELEASE_DIR, "driver",
"lib")
utility.make_directory(dynamic_library_dest_path)
shutil.copy2(dynamic_library_path, dynamic_library_dest_path)
def copy_windows_lib(softdevice_name='s130'):
import_library_name = softdevice_name + '_' + config.LIB_NAME + config.IMPORT_LIB_SUFFIX
import_library_path = posixpath.join(config.BUILD_DIR, 'driver',
import_library_name)
release_import_library_path = posixpath.join(config.RELEASE_DIR, 'driver', 'lib')
utility.make_directory(release_import_library_path)
shutil.copy2(import_library_path, release_import_library_path)
def copy_windows_lib(softdevice_name='s130'):
import_library_name = softdevice_name + '_' + config.LIB_NAME + config.IMPORT_LIB_SUFFIX
import_library_path = posixpath.join(config.BUILD_DIR, 'driver',
import_library_name)
release_import_library_path = posixpath.join(config.RELEASE_DIR, 'driver',
'lib')
utility.make_directory(release_import_library_path)
shutil.copy2(import_library_path, release_import_library_path)
def write_jpeg(app_segment_data, jpeg_data_list, output_file_path):
# ep = ">"
byte_order = 0x4d4d
ep = ENDIAN_TYPE[byte_order]
utility.make_directory(output_file_path)
with open(output_file_path, "wb") as f:
data = struct.pack("%sH" % ep, SOI_TAG)
f.write(data)
app_segment_data.write_segment_header(f, ep, byte_order)
for jpeg_data in jpeg_data_list:
f.write(jpeg_data)
def create_image_middle_jpg(input_file_path, output_file_path):
if os.path.isfile(output_file_path):
return
print("{} -> {}".format(input_file_path, output_file_path))
img = Image.open(input_file_path)
img.resize((800, 800), Image.ANTIALIAS)
utility.make_directory(output_file_path)
img.save(output_file_path)
def create_image_small_jpg(input_file_path, output_file_path):
if os.path.isfile(output_file_path):
return
print("{} -> {}".format(input_file_path, output_file_path))
img = Image.open(input_file_path)
img.thumbnail((160, 160), Image.ANTIALIAS)
utility.make_directory(output_file_path)
img.save(output_file_path)
def execute_ffmpeg(url, output_file_path):
if os.path.isfile(output_file_path):
return
utility.make_directory(output_file_path)
command = (settings.FFMPEG_PATH, "-i", url, "-y", "-movflags", "faststart",
"-c", "copy", "-bsf:a", "aac_adtstoasc", output_file_path)
print(command)
def call_back(returncode, stdout_message):
print(returncode, stdout_message)
pass
utility.create_process(command, call_back, 1, 2400)()
def open(self):
make_directory(self.trace_dir)
self.alpha_writer = ConcentrationParameterWriter(self.trace_dir)
self.labels_writer = LabelsWriter(self.trace_dir, self.mutation_ids)
self.cellular_frequency_writers = {}
for sample_id in self.sample_ids:
self.cellular_frequency_writers[sample_id] = CellularFrequenciesWriter(self.trace_dir,
sample_id,
self.mutation_ids)
if self.update_precision:
self.precision_writer = PrecisionWriter(self.trace_dir)
def open(self):
make_directory(self.trace_dir)
self.alpha_writer = ConcentrationParameterWriter(self.trace_dir)
self.labels_writer = LabelsWriter(self.trace_dir, self.mutation_ids)
self.cellular_frequency_writers = {}
for sample_id in self.sample_ids:
self.cellular_frequency_writers[
sample_id] = CellularFrequenciesWriter(self.trace_dir,
sample_id,
self.mutation_ids)
if self.update_precision:
self.precision_writer = PrecisionWriter(self.trace_dir)
def get_iframe_html(opener, url, session_id):
referer_url = settings.URL_ROOT
req = urllib.request.Request(url)
req.add_header('Referer', referer_url)
req.add_header('accept', 'application/json, text/javascript, */*; q=0.01')
req.add_header('accept-language', settings.HTTP_HEADER_LANGUAGE)
req.add_header('User-Agent', settings.HTTP_HEADER_USER_AGENT)
res = opener.open(req)
file_path = os.path.join(settings.PICTURE_TEMP_PATH,
"{}_iframe.html".format(session_id))
res_data = res.read()
utility.make_directory(file_path)
with open(file_path, mode="wb") as f:
f.write(res_data)
soup = BeautifulSoup(res_data, "html.parser")
video_element = soup.find("video")
if video_element is None:
print("video element not found -> id:{}".format(session_id))
return
movie_url_element = video_element.find("source")
if movie_url_element is None:
print("video source element not found -> id:{}".format(session_id))
return
movie_url_value = movie_url_element.get("src")
print(movie_url_value)
parse_result = urllib.parse.urlparse(movie_url_value)
scheme, netloc, url_path, _, _, _ = parse_result
hd_2000k_url = url = urllib.parse.urlunparse(
(scheme, netloc, url_path.replace("index.m3u8",
"hd_2000k_.m3u8"), None, None, None))
file_path = os.path.join(settings.PICTURE_TEMP_PATH, "strage",
"{}.mp4".format(session_id))
execute_ffmpeg(hd_2000k_url, file_path)
def combine_file(file_path_list, target_file_path, mergin, v_or_h,
target_override_date):
if v_or_h == COMBINE_V:
height_name = "height"
width_name = "width"
elif v_or_h == COMBINE_H:
height_name = "width"
width_name = "height"
def calc_total_height(height_name, width_name):
total_height = 0
width = getattr(im_list[0], width_name)
for im in im_list:
total_height = total_height + getattr(im, height_name)
if width != getattr(im, width_name):
raise Error("width difference {} != {}".format(
width, getattr(im, width_name)))
return total_height, width
im_list = []
for file_path in file_path_list:
im_list.append(Image.open(file_path))
mergin_total = mergin * (len(file_path_list) - 1)
total_height, width = calc_total_height(height_name, width_name)
if v_or_h == COMBINE_V:
target_image = Image.new("RGB", (width, total_height - mergin_total))
elif v_or_h == COMBINE_H:
target_image = Image.new("RGB", (total_height - mergin_total, width))
current_height = 0
for im in im_list:
if v_or_h == COMBINE_V:
target_image.paste(im, (0, current_height))
elif v_or_h == COMBINE_H:
target_image.paste(im, (current_height, 0))
current_height = current_height + getattr(im, height_name) - mergin
utility.make_directory(target_file_path)
target_image.save(target_file_path)
override_date(target_file_path, target_override_date)
def __init__(self, configuration):
utility.Print('python_info',
'\nCreated instance of HistogramHandler class')
# Get campaign name
self.campaigns = configuration.general.campaigns
self.campaigns_info = configuration.campaigns.info
# Force all regardless of already existing files
self.force_all = configuration.general.force_all
# Work with local files or remote
self.work_locally = configuration.general.work_locally
# ------ Paths -------
self.path_main = configuration.paths.main
self.path_samples = configuration.paths.samples
self.path_logical_file_names = configuration.paths.logical_file_names
self.path_batch_results = utility.make_directory(
configuration.paths.batch_results, 'histograms')
self.path_batch_templates = utility.make_directory(
configuration.paths.batch_templates, 'histograms')
self.path_histograms = utility.make_directory(
configuration.paths.histograms)
self.path_proxy = configuration.paths.proxy
# ------ Batch options -------
self.batch_type = configuration.general.batch_type
self.number_of_jobs = configuration.general.number_of_jobs
self.number_of_files = configuration.general.number_of_files
self.send_jobs = configuration.general.send_jobs
self.batch_templates = configuration.general.batch_templates[
'histogram']
self.queue_lxbatch = configuration.general.queue_lxbatch
# ------ Samples -------
self.samples_info = configuration.samples.info
# ---- btag configuration ----
self.config_btagvalidation = configuration.btagvalidation.parameters
# ------ Browsing options -------
self.remote_locations = configuration.general.remote_locations
def create_movie_thumbnail(input_file_path, output_file_path):
if os.path.isdir(os.path.dirname(output_file_path)):
pass
else:
utility.make_directory(output_file_path)
if os.path.isfile(output_file_path):
return
command = (cf.FFMPEG_PATH, "-y", "-i", input_file_path, "-vf",
"fps=1/10,scale=480:-1", output_file_path)
print(command)
def call_back(returncode, stdout_message):
print(returncode, stdout_message)
pass
utility.create_process(command, call_back, 1, 20)()
def __init__(self,
path_templates,
templates,
arguments,
batch,
queue=None):
self.batch = batch
self.path_templates = path_templates
self.templates = templates
self.arguments = arguments
self.queue = queue
# Add aditional layer of directories for each job
_new_path_batch = self.arguments['<path_batch_file_wo_ext>']
self.arguments['<path_batch_file_wo_ext>'] = _new_path_batch.replace(
self.arguments['<path_batch>'], _new_path_batch)
self.arguments['<path_batch>'] = _new_path_batch
# make batch directory
utility.make_directory(self.arguments['<path_batch>'])
def _create_info_file(root_path, json_encoder, file_info_list_by_date, cls):
output_file_path = os.path.join(root_path, INFO_FILE_NAME)
utility.make_directory(output_file_path)
date_list = [FolderInfo(v) for v, _ in file_info_list_by_date.items()]
write_data = json.dumps(
date_list,
cls=json_encoder,
indent=" ",
)
with open(output_file_path, "w") as fp:
fp.write(write_data)
for one_day, file_info_list in file_info_list_by_date.items():
output_file_path = os.path.join(root_path, one_day,
cls.get_directory_name(),
INFO_FILE_NAME)
utility.make_directory(output_file_path)
write_data = json.dumps(
file_info_list,
cls=json_encoder,
indent=" ",
)
with open(output_file_path, "w") as fp:
fp.write(write_data)
for root, dirs, files in os.walk(root_path):
if not dirs:
continue
folder_list = [FolderInfo(os.path.basename(v)) for v in dirs]
write_data = json.dumps(
folder_list,
cls=json_encoder,
indent=" ",
)
with open(os.path.join(root, INFO_FILE_NAME), "w") as fp:
fp.write(write_data)
def login(login_id, password, local_test_flag, do_login_flag, root_path):
cookie_file_path = os.path.join(root_path, "cookie.txt")
if local_test_flag:
return None
utility.make_directory(cookie_file_path)
if os.path.isfile(cookie_file_path) == False:
with open(cookie_file_path, "w") as fp:
fp.write("# Netscape HTTP Cookie File")
cookie_jar = http.cookiejar.MozillaCookieJar(cookie_file_path)
cookie_jar.load()
cookie_processor = urllib.request.HTTPCookieProcessor(cookie_jar)
opener = urllib.request.build_opener(cookie_processor)
if do_login_flag:
phrase = get_root_html(opener, root_path)
login_main(opener, login_id, password, phrase)
return opener
def copy_hex_files(softdevice_name='s130'):
hex_file_list = []
version = {
's110': '7.1.0',
's120': '1.0.1',
's130': '1.0.0'}[softdevice_name]
hex_resource_path = config.RESOURCE_DIR
hex_file_name = 'connectivity_115k2_with_{0}_{1}.hex'.format(softdevice_name, version)
file_list_append(hex_file_list, hex_resource_path, hex_file_name)
if not config.PLATFORM_SYSTEM == 'Darwin':
hex_file_name = 'connectivity_1m_with_{0}_{1}.hex'.format(softdevice_name, version)
file_list_append(hex_file_list, hex_resource_path, hex_file_name)
release_hex_path = posixpath.join(config.RELEASE_DIR, 'firmware')
utility.make_directory(release_hex_path)
for hex_file_path in hex_file_list:
shutil.copy2(hex_file_path, release_hex_path)
def get_session_html(opener, url):
referer_url = settings.URL_ROOT
req = urllib.request.Request(url)
req.add_header('Referer', referer_url)
req.add_header('accept', 'application/json, text/javascript, */*; q=0.01')
req.add_header('accept-language', settings.HTTP_HEADER_LANGUAGE)
req.add_header('User-Agent', settings.HTTP_HEADER_USER_AGENT)
res = opener.open(req)
session_id = url.split(r"/")[-1]
file_path = os.path.join(settings.PICTURE_TEMP_PATH,
"{}.html".format(session_id))
res_data = res.read()
utility.make_directory(file_path)
with open(file_path, mode="wb") as f:
f.write(res_data)
soup = BeautifulSoup(res_data, "html.parser")
iframe_url_element = soup.find("iframe",
{"class": "frame embed-responsive-item"})
if iframe_url_element is None:
print("iframe not found -> id:{}".format(session_id))
return
iframe_url_value = iframe_url_element.get("src")
# print(iframe_url_value)
get_iframe_html(opener, iframe_url_value, session_id)
#PBS -S /bin/bash
source /home/jswinban/sw/init.sh
source /home/jswinban/sw/lofim/lofarinit.sh
cd %s
time python /home/jswinban/imaging/imaging-multibeam.py %s
"""
TEMPLATE_JOB = textwrap.dedent(TEMPLATE_JOB).strip()
if __name__ == "__main__":
target_obsid = sys.argv[1]
cal_obsid = sys.argv[2]
template_parset = sys.argv[3]
CAL_OUTPUT = os.path.join(OUTPUT_DIR, "calibrator", cal_obsid)
TARGET_OUTPUT = os.path.join(OUTPUT_DIR, "target", target_obsid)
make_directory(CAL_OUTPUT)
make_directory(TARGET_OUTPUT)
# Check data exists: we should have sum(BAND_SIZE) subbands in each beam,
# N_BEAMS beams per target_obsid, and 1 beam per cal_obsid.
# We write the validated data to input files for the imaging pipeline.
ms_list = sorted_ms_list(os.path.join(INPUT_DIR, cal_obsid))[:sum(BAND_SIZE)]
assert(len(ms_list) == sum(BAND_SIZE))
with open(os.path.join(TARGET_OUTPUT, "cal_ms_list"), 'w') as f:
for ms in ms_list:
f.write("%s\n" % ms)
ms_list = sorted_ms_list(os.path.join(INPUT_DIR, target_obsid))[:sum(BAND_SIZE)*N_BEAMS]
assert(len(ms_list) == sum(BAND_SIZE) * N_BEAMS)
with open(os.path.join(TARGET_OUTPUT, "target_ms_list"), 'w') as f:
for ms in ms_list:
# Our single command line argument is a parset containing all
# configuration information we'll need.
input_parset = lofar.parameterset.parameterset(sys.argv[1])
# We require `sbs_per_beam` input MeasurementSets for each beam, including
# the calibrator.
sbs_per_beam = sum(input_parset.getIntVector("band_size"))
print "Locating calibrator data and checking paths"
ms_cal = {}
ms_cal["datafiles"] = read_ms_list(input_parset.getString("cal_ms_list"))
assert len(ms_cal["datafiles"]) == sbs_per_beam
ms_cal["output_dir"] = os.path.join(
input_parset.getString("output_dir"), "calibrator", input_parset.getString("cal_obsid")
)
make_directory(ms_cal["output_dir"])
print "Copying calibrator subbands to output"
ms_cal["datafiles"] = copy_to_work_area(ms_cal["datafiles"], ms_cal["output_dir"])
print "Locating target data and checking paths"
# ms_target will be a dict that provides all the information we need to
# process each independent element of the observation, where an "element"
# is a combination of a beam (SAP) and a band (number of subbands)
ms_target = {}
target_mss = read_ms_list(input_parset.getString("target_ms_list"))
assert len(target_mss) == input_parset.getInt("n_beams") * sbs_per_beam
for beam, data in enumerate(zip(*[iter(target_mss)] * sbs_per_beam)):
start_sb = 0
def copy_files_all_samples_locally(self):
utility.Print('python_info',
'\nCalled copy_files_all_samples_locally function.')
_batch_templates_copy = self.batch_templates['copy']
_path_batch_results_copy = utility.make_directory(
self.path_batch_results, 'copy')
_path_batch_templates_copy = utility.make_directory(
self.path_batch_templates, 'copy')
# Loop over samples
for _s in self.samples:
utility.Print('status', '\nSample: {0}'.format(_s))
# Loop over all remote locations
for _l in self.remote_locations['path']:
# Loop over all subsamples
for _ss in self.samples_info[_s]['subsample'].values():
_file_missing = os.path.join(
self.path_logical_file_names, 'local', _l, _s,
_ss.replace('/', '__') + '_missing.txt')
_file_missing = open(_file_missing, 'r')
for _n, _f in enumerate(_file_missing):
_f = _f.rstrip()
# Number of jobs handler
if _n >= self.number_of_jobs and not self.number_of_jobs == -1:
continue
if os.path.isfile(_f):
utility.Print(
'python_info',
'File {0} already exists.'.format(_f))
continue
# To create batch jobs, one needs to edit template script which is done below
_path_remote_samples = os.path.join(
self.remote_locations['storage_element'][_l],
self.remote_locations['path'][_l])
_x = _f.split(
'/') # split input file into chunks wrt to '/'
_source = _f.replace(
self.path_samples,
_path_remote_samples) # place from where to copy
_destination = '/'.join(_x[:-1]) # path where to copy
_batch_file_wo_ext = _x[-1].replace(
'.root', '') # file name without any extension
_path_batch = _destination.replace(
self.path_samples, _path_batch_results_copy
) # path to newly created batch script
_path_batch_file_wo_ext = os.path.join(
_path_batch, _batch_file_wo_ext
) # path and name of the script without any extension
_protocol = self.remote_locations['protocol'][
_l] # protocol used for copying
_path_proxy = self.path_proxy # one needs proxy file to give permission to condor
_file_proxy = self.path_proxy.split('/')[-1]
_batch_arguments = {
'<path_batch>':
_path_batch,
'<path_batch_file_wo_ext>':
_path_batch_file_wo_ext,
'<source>':
_source,
'<destination>':
_destination,
'<protocol>':
_protocol,
'<X509_USER_PROXY_path>':
_path_proxy,
'<X509_USER_PROXY_file>':
_file_proxy,
'<SCRAM_ARCH>':
os.environ['SCRAM_ARCH'],
'<CMSSW_dir>':
os.path.join(os.environ['CMSSW_BASE'], 'src'),
}
# Create batch tool instance
_batch = utility.BatchTool(_path_batch_templates_copy,
_batch_templates_copy,
_batch_arguments,
self.batch_type)
# Make scripts first
_batch.make_scripts()
# Send jobs
if self.send_jobs:
_batch.send_job()
_file_missing.close()
def check_missing_files_all_samples_locally(self):
''' Loop over missing file and remove any file which exists and it has error.'''
utility.Print(
'python_info',
'\nCalled check_missing_files_all_samples_locally function.')
_batch_templates_check = self.batch_templates['check']
_path_batch_results_check = utility.make_directory(
self.path_batch_results, 'check')
_path_batch_templates_check = utility.make_directory(
self.path_batch_templates, 'check')
# Loop over samples
for _s in self.samples:
utility.Print('status', '\nSample: {0}'.format(_s))
# Loop over all remote locations
for _l in self.remote_locations['path']:
# Loop over all subsamples
for _ss in self.samples_info[_s]['subsample'].values():
_file_missing = os.path.join(
self.path_logical_file_names, 'local', _l, _s,
_ss.replace('/', '__') + '_missing.txt')
_file_missing = open(_file_missing, 'r')
for _f in _file_missing:
_f = _f.rstrip()
# Send check files on batch
if self.send_jobs:
_x = _f.split(
'/') # split input file into chunks wrt to '/'
_batch_file_wo_ext = _x[-1].replace(
'.root', '') # file name without any extension
_destination = '/'.join(_x[:-1])
_path_batch = _destination.replace(
self.path_samples, _path_batch_results_check
) # path to newly created batch script
_path_batch_file_wo_ext = os.path.join(
_path_batch, _batch_file_wo_ext
) # path and name of the script without any extension
_path_proxy = self.path_proxy # one needs proxy file to give permission to condor
_file_proxy = self.path_proxy.split('/')[-1]
_batch_arguments = {
'<path_batch>':
_path_batch,
'<path_batch_file_wo_ext>':
_path_batch_file_wo_ext,
'<file_name>':
_f,
'<X509_USER_PROXY_path>':
_path_proxy,
'<X509_USER_PROXY_file>':
_file_proxy,
'<SCRAM_ARCH>':
os.environ['SCRAM_ARCH'],
'<CMSSW_dir>':
os.path.join(os.environ['CMSSW_BASE'], 'src'),
}
# Create batch tool instance
_batch = utility.BatchTool(
_path_batch_templates_check,
_batch_templates_check, _batch_arguments,
self.batch_type)
# Make scripts first
_batch.make_scripts()
# Send job
_batch.send_job()
else:
if os.path.isfile(_f):
if utility.check_root_file(_f):
utility.Print('status',
'File OK {0}'.format(_f))
else:
utility.Print(
'error',
'Error with {0}. Removing ...'.format(
_f))
sp.check_output(['rm', _f])
_file_missing.close()
def save_logical_file_names_all_samples_remotely(self):
'''
Tries to find all the files for the corresponding samples and saves them in .txt
NOTE: this function overwrites files every time.
'''
utility.Print(
'python_info',
'\nCalled save_logical_file_names_all_samples function.')
# Loop over samples
for _s in self.samples:
utility.Print('status', '\nSample: {0}'.format(_s))
# Loop over all remote locations
for _l in self.remote_locations['path']:
utility.make_directory(self.path_logical_file_names, 'remote',
_l, _s)
# Browse for the sample files
try:
# Get LFNs
_remote_path = os.path.join(
self.remote_locations['storage_element'][_l],
self.remote_locations['path'][_l], _s)
_remote_lfns = self._wrapper_gfal_ls_r(
_remote_path, self.remote_locations['protocol'][_l])
# Filter logical files names so that only interesting ones pass
_keywords_any = self.samples_info[_s]['subsample'].values(
) # filter lfn which has subsample string
_remote_lfns = [
_ll for _ll in _remote_lfns if utility.filter_keywords(
_ll, self.search_keywords['all'],
self.search_keywords['any'] +
_keywords_any, self.search_keywords['none'])
]
# Group files according to subsample and save them in separate files
_remote_lfns = {
_ss: filter(lambda x: '/' + _ss + '/' in x,
_remote_lfns)
for _ss in self.samples_info[_s]['subsample'].values()
}
# Save into files
for _ss, _r in _remote_lfns.iteritems():
_file = os.path.join(self.path_logical_file_names,
'remote', _l, _s,
_ss.replace('/', '__') + '.txt')
# Save to a file
with open(_file, 'w') as _output:
_output.write('\n'.join(_r))
utility.Print(
'status',
'The LFNs were written to "{}".'.format(_file))
# If something gets wrong notify
except Exception, e:
utility.Print(
'error',
"Error with sample {0} at {1}: {2}".format(_s, _l, e))
def get_picture_list(opener, eventno, categoryno, offset=0):
file_path = os.path.join(
settings.PICTURE_TEMP_PATH, "json_data", eventno,
"category_picture_list_{}_{}.json".format(categoryno, offset))
json_data = None
if os.path.isfile(file_path):
with open(file_path, mode="r") as f:
try:
json_data = json.loads(f.read())
except:
print("json read error ({})".format(file_path))
if json_data is None:
parse_result = urllib.parse.urlparse(settings.URL_ROOT)
query_dict = {
'action_Api_FastViewer_PhotoList': 'true',
'eventno': eventno,
'categoryno': categoryno,
'offset': offset,
'page': 'photolist',
'sortkey': '0',
}
scheme, netloc, path, params, _, fragment = parse_result
query = urllib.parse.urlencode(query_dict)
url = urllib.parse.urlunparse(
(scheme, netloc, path, params, query, fragment))
print(url)
req = urllib.request.Request(url)
referer_url = '{}/?action_user_FastViewer=t&eventno={}'.format(
settings.URL_ROOT, eventno)
req.add_header('Referer', referer_url)
req.add_header('accept',
'application/json, text/javascript, */*; q=0.01')
req.add_header('accept-language', settings.HTTP_HEADER_LANGUAGE)
req.add_header('User-Agent', settings.HTTP_HEADER_USER_AGENT)
res = opener.open(req)
utility.make_directory(file_path)
with open(file_path, mode="wb") as f:
f.write(res.read())
with open(file_path, mode="r") as f:
json_data = json.loads(f.read())
json_data_list = []
json_data_list.append(json_data)
if json_data["message"]["pager"]["hasnext"]:
next_json_data_list = get_picture_list(opener,
eventno,
categoryno,
offset=offset + 100)
json_data_list += next_json_data_list
return json_data_list
# Our single command line argument is a parset containing all
# configuration information we'll need.
input_parset = lofar.parameterset.parameterset(sys.argv[1])
# We require `sbs_per_beam` input MeasurementSets for each beam, including
# the calibrator.
sbs_per_beam = sum(input_parset.getIntVector("band_size"))
print "Locating calibrator data and checking paths"
ms_cal = {}
ms_cal["datafiles"] = read_ms_list(input_parset.getString("cal_ms_list"))
assert (len(ms_cal["datafiles"]) == sbs_per_beam)
ms_cal["output_dir"] = os.path.join(input_parset.getString("output_dir"),
"calibrator",
input_parset.getString("cal_obsid"))
make_directory(ms_cal["output_dir"])
print "Copying calibrator subbands to output"
ms_cal["datafiles"] = copy_to_work_area(ms_cal["datafiles"],
ms_cal["output_dir"])
print "Locating target data and checking paths"
# ms_target will be a dict that provides all the information we need to
# process each independent element of the observation, where an "element"
# is a combination of a beam (SAP) and a band (number of subbands)
ms_target = {}
target_mss = read_ms_list(input_parset.getString("target_ms_list"))
assert (len(target_mss) == input_parset.getInt("n_beams") * sbs_per_beam)
for beam, data in enumerate(zip(*[iter(target_mss)] * sbs_per_beam)):
