def main():
parser = argparse.ArgumentParser("Inject a file into a plugin/step")
parser.add_argument("filepath", type=str, help="filepath to inject")
parser.add_argument("--plugin",
type=str,
help="plugin name (default :guess_file_type)",
default="guess_file_type")
parser.add_argument("--step",
type=str,
help="step name (default: main)",
default="main")
parser.add_argument("--move",
action="store_true",
help="move the file instead of copying it "
"(default: copy)")
parser.add_argument("--random-basename",
action="store_true",
help="use a random basename for copying/moving "
"the file (default: keep the original basename)")
parser.add_argument(
"--incoming",
action="store_true",
help="ignore plugin and step parameter and inject "
"the file into the first configured directory listened"
" by the MFDATA_INTERNAL_PLUGINS_WATCHED_DIRECTORIES env var")
args = parser.parse_args()
try:
x = XattrFile(args.filepath)
except Exception:
logger.warning("can't open %s" % args.filepath)
sys.exit(1)
if args.random_basename:
basename = get_unique_hexa_identifier()
else:
basename = os.path.basename(args.filepath)
if args.incoming:
env_var = 'MFDATA_INTERNAL_PLUGINS_WATCHED_DIRECTORIES'
first_directory = os.environ[env_var].split(',')[0]
new_filepath = os.path.join(os.environ['MFDATA_DATA_IN_DIR'],
first_directory, basename)
else:
new_filepath = \
os.path.join(get_plugin_step_directory_path(args.plugin,
args.step), basename)
if args.move:
res, moved = x.move_or_copy(new_filepath)
if not res:
logger.warning("can't move %s to %s" %
(args.filepath, new_filepath))
sys.exit(1)
else:
res = x.copy_or_nothing(new_filepath)
if not res:
logger.warning("can't copy %s to %s" %
(args.filepath, new_filepath))
sys.exit(1)
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument(
'filepath', help='Path of the file of which you want to print tags')
parser.add_argument('name', help='name of the tag')
parser.add_argument('value', help='value of the tag')
args = parser.parse_args()
xaf = XattrFile(args.filepath)
xaf.tags[args.name] = args.value
xaf.commit()
def make_tmp_xattrfile():
foo = AcquisitionTestStep1()
tmp_filepath = foo.get_tmp_filepath()
with open(tmp_filepath, "wb") as f:
f.write("foo\n".encode('utf8'))
f.write("bar\n".encode('utf8'))
return XattrFile(tmp_filepath)
def process(self, xaf):
if self.args.keep_original_basenames:
new_filepath = os.path.join(self.args.dest_dir,
self.get_original_basename(xaf))
else:
new_filepath = os.path.join(self.args.dest_dir, xaf.basename())
fcmi = int(self.args.force_chmod, 8) \
if self.args.force_chmod is not None else None
# Store old xaf filepath to display in the logs
old_filepath = xaf.filepath
if self.drop_tags:
xaf.clear_tags()
else:
self._set_after_tags(xaf, True)
success, moved = xaf.move_or_copy(new_filepath, chmod_mode_int=fcmi)
if success:
if moved:
self.info("%s moved into %s", old_filepath, new_filepath)
else:
self.info("%s copied into %s", xaf.filepath, new_filepath)
if self.keep_tags:
tags_filepath = new_filepath + self.keep_tags_suffix
xaf.write_tags_in_a_file(tags_filepath)
XattrFile(new_filepath).clear_tags()
return True
else:
self.warning("Can't move/copy %s to %s", xaf.filepath,
new_filepath)
return False
def _no_match(self, xaf):
self.info("No condition matched for %s" % xaf.filepath)
if self.no_match_policy == "keep":
new_filepath = \
os.path.join(_get_or_make_trash_dir(self.plugin_name,
"nomatch"),
xaf.basename())
old_filepath = xaf.filepath
success, moved = xaf.move_or_copy(new_filepath)
if success:
if moved:
self.info("%s moved into %s", old_filepath, new_filepath)
else:
self.info("%s copied into %s", xaf.filepath, new_filepath)
if self.args.no_match_policy_keep_keep_tags:
tags_filepath = new_filepath + \
self.args.no_match_policy_keep_keep_tags_suffix
xaf.write_tags_in_a_file(tags_filepath)
XattrFile(new_filepath).clear_tags()
elif self.no_match_policy == "move":
new_filepath = \
os.path.join(self.args.no_match_policy_move_dest_dir,
xaf.basename())
xaf.move_or_copy(new_filepath)
return True
def destroy(self):
self.debug("destroy called")
# we generate a IN_MOVE event on remaining files to generate new
# events on redis (to be sure that remaining files will be checked
# at startup)
filepaths = list(self.__xafs.keys())
for filepath in filepaths:
try:
xaf = self.__xafs[filepath][2]
except KeyError:
pass
new_filepath = filepath + ".t"
self.debug("move %s to %s" % (filepath, new_filepath))
xaf.rename(new_filepath)
xaf2 = XattrFile(new_filepath)
self.debug("move %s to %s" % (new_filepath, filepath))
xaf2.rename(filepath)
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument(
'filepath', help='Path of the file of which you want to print tags')
parser.add_argument('name', help='name of the tag')
args = parser.parse_args()
xaf = XattrFile(args.filepath)
if args.name in xaf.tags:
print(xaf.tags[args.name].decode('utf-8'))
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument(
'filepath', help='Path of the file of which you want to print tags')
args = parser.parse_args()
xaf = XattrFile(args.filepath)
keys = sorted(xaf.tags.keys())
for key in keys:
print("%s = %s" % (key.decode('utf8'), xaf.tags[key].decode('utf8')))
def image_crop_and_export(self, input_file, x=0, y=0, w=10, h=10):
try:
x, y, w, h = int(x), int(y), int(w), int(h)
except ValueError as e:
print(e, "(wrong crop var type)")
# Cropping part
try:
imageObject = Image.open(input_file.filepath)
except IOError:
self.error("IOError: Can't open %s (%s)" %
(input_file.filepath, self.original_file_name))
return 1
if x + y + w + h:
output = "cropped_%s" % (self.original_file_name)
try:
cropped = imageObject.crop((x, y, w, h))
except OSError:
truncated_file_name = ("truncated_%s" %
(self.original_file_name))
imageObject.save("/home/mfdata/plugins/image_treatment/"
"files/truncated_files/%s" %
truncated_file_name)
imageObject.close()
self.error("OSError truncated file %s"
"sent to truncated_files dir" % truncated_file_name)
return 1
cropped.save(output, format="jpeg")
cropped.save("/home/mfdata/plugins/image_treatment/files/cropped/"
"%s" % self.original_file_name,
format="jpeg")
cropped.close()
# Xattr part
output_attr = XattrFile(output)
for key in input_file.tags:
output_attr.tags[key] = input_file.tags[key]
if all(key in input_file.tags
for key in (b"crop_x", b"crop_y", b"crop_width",
b"cropp_height")):
del input_file.tags[b"crop_x"]
del input_file.tags[b"crop_y"]
del input_file.tags[b"crop_width"]
del input_file.tags[b"crop_heigth"]
output_attr.tags["actions"] = ",".join(
(output_attr.tags["actions"].decode('utf-8')).split(",")[1:])
output_attr.commit()
output_attr.move_or_copy("/home/mfdata/var/in/incoming/%s" %
(output))
else:
print("No crop options")
imageObject.close()
def add_tags_and_copy(self, name, file_path, date):
"""
Add tags to the given file.
Attention, json tag can't have anything different from strings
Use load_parameters_from to get dict in json.
"""
tagger = XattrFile(file_path)
tagger.tags["date"] = date
self.load_parameters(tagger, name)
tagger.commit()
tagger.copy("/home/mfdata/var/in/incoming/%s" % file_path[10:])
def getFirstFrame(self, at):
at.tags["actions"] = \
",".join((at.tags["actions"].decode('utf-8')).split(",")[1:])
vidcap = cv2.VideoCapture(at.filepath)
success, image = vidcap.read()
if success:
cv2.imwrite("frame.jpg", image)
vidcap.release()
output_attr = XattrFile("frame.jpg")
for key in at.tags:
output_attr.tags[key] = at.tags[key]
output_attr.commit()
output_attr.move_or_copy("/home/mfdata/var/in/incoming/test")
def process(self, xaf):
"""
This function:
- Convert a GRIB file into a NetCDF file.
- Read some data of the NetCDF file
:param xaf: the input GRIB data file as an XattrFile object
:return: True, if the process is successful, False, if the process failed
"""
# xaf.filepath is the internal file name created by the switch plugin into a temporary directory
self.info("process for file %s" % xaf.filepath)
try:
# In order to get the original GRIB file name, call AcquisitionStep.get_original_basename
original_grib_filename = str(AcquisitionStep.get_original_basename(self, xaf))
# Build the output NetCDF file name from the input file name,
netcdf_filename = re.sub(r'(\.grb|\.grib2|\.grib)$', '', str(original_grib_filename)) + ".nc"
netcdf_filepath = os.path.normpath(os.path.join(self.args.netcdf_dest_dir, netcdf_filename))
# Convert Grib to Netcdf
self.grib_to_netcdf_command(xaf.filepath, netcdf_filepath)
# We tags/attributes in a specific file
if self.args.keep_tags:
tags_filepath = netcdf_filepath + self.args.keep_tags_suffix
xaf.write_tags_in_a_file(tags_filepath)
XattrFile(netcdf_filepath).clear_tags()
# Read the output NetCDF
# Log the dimensions name and variable names
netcdf_dataset = Dataset(netcdf_filepath, "r")
self.info("Dimensions of the Netcdf dataset {}:".format(netcdf_filepath))
for dim_name in netcdf_dataset.dimensions:
self.info(dim_name)
self.info("Variables of the Netcdf dataset {}:".format(netcdf_filepath))
for var_name in netcdf_dataset.variables:
self.info(var_name)
except Exception as e:
self.exception(str(e))
return False
return True
def process(self, xaf):
original_dirname = self.get_original_dirname(xaf)
original_basename = self.get_original_basename(xaf)
original_uid = self.get_original_uid(xaf)
random_basename = get_unique_hexa_identifier()
step_counter = self._get_counter_tag_value(xaf, not_found_value='999')
if step_counter != 999 and step_counter != 0:
step_counter_minus_1 = step_counter - 1
else:
step_counter_minus_1 = step_counter
rendered_template = os.path.join(self.archive_dir,
self.strftime_template)
rendered_template = rendered_template.replace('{RANDOM_ID}',
random_basename)
rendered_template = rendered_template.replace('{ORIGINAL_BASENAME}',
original_basename)
rendered_template = rendered_template.replace('{ORIGINAL_UID}',
original_uid)
rendered_template = rendered_template.replace('{ORIGINAL_DIRNAME}',
original_dirname)
rendered_template = rendered_template.replace('{STEP_COUNTER}',
str(step_counter))
rendered_template = \
rendered_template.replace('{STEP_COUNTER_MINUS_1}',
str(step_counter_minus_1))
new_filepath = time.strftime(rendered_template)
dirname = os.path.dirname(new_filepath)
res = mkdir_p(dirname)
if not res:
self.warning("can't mkdir %s" % dirname)
return False
# Store old xaf filepath to display in the logs
old_filepath = xaf.filepath
success, moved = xaf.move_or_copy(new_filepath)
if success:
if moved:
self.info("%s moved into %s", old_filepath, new_filepath)
else:
self.info("%s copied into %s", xaf.filepath, new_filepath)
if self.keep_tags:
tags_filepath = new_filepath + self.keep_tags_suffix
xaf.write_tags_in_a_file(tags_filepath)
XattrFile(new_filepath).clear_tags()
return True
else:
self.warning("Can't move/copy %s to %s", xaf.filepath,
new_filepath)
return False
def _keep(self, xaf):
new_filepath = os.path.join(
_get_or_make_trash_dir(self.plugin_name, "nomatch"),
xaf.basename(),
)
old_filepath = xaf.filepath
success, moved = xaf.move_or_copy(new_filepath)
if success:
if moved:
self.info("%s moved into %s", old_filepath, new_filepath)
else:
self.info("%s copied into %s", xaf.filepath, new_filepath)
tags_filepath = new_filepath + ".tags"
xaf.write_tags_in_a_file(tags_filepath)
XattrFile(new_filepath).clear_tags()
self.add_trace(xaf, ">nomatch", "keep")
def make_xattrfile(filepath):
return XattrFile(filepath, get_redis_callable=unittests_get_redis_callable)
def make_tmp_xattrfile():
foo = AcquisitionDeleteTestStep()
tmp_filepath = foo.get_tmp_filepath()
with open(tmp_filepath, 'w+') as f:
f.write("foo\n")
return XattrFile(tmp_filepath)
def test_redis_empty_after_preprocessing(self):
"""Test if there are no left over keys in Redis"""
# xattrf only used to access Redis
xattrf = XattrFile(os.path.realpath(__file__))
r = xattrf._get_redis()
self.assertEquals(len(r.keys()), 0)