def upload(self, remote, local, force=False): """ Upload a local file to the store :param str remote: name in the server :param str local: local file name :param bool force: (default *False*) if the file already exists and has the same digest, do not re-upload it. """ fl = self.list([remote]) if force == False and remote in fl: remote_hash = fl[remote] h = hashlib.sha256() commonl.hash_file(h, local) if remote_hash == h.hexdigest(): # remote hash is the same, no need to upload return with io.open(local, "rb") as inf: self.target.ttbd_iface_call("store", "file", method="POST", file_path=remote, files={'file': inf})
def get_list(_target, _who, _args, _files, user_path): file_data = {} for path, _dirnames, filenames in os.walk(user_path): for filename in filenames: file_path = path + "/" + filename h = hashlib.sha256() commonl.hash_file(h, file_path) file_data[file_path[len(user_path) + 1:]] = h.hexdigest() return dict(result=file_data)
def eval_04(self, linux): linux.ssh.copy_to(__file__) base_file = os.path.basename(__file__) copied_file = os.path.join(self.tmpdir, base_file) linux.ssh.copy_from(base_file, copied_file) orig_hash = commonl.hash_file(hashlib.sha256(), __file__) copied_hash = commonl.hash_file(hashlib.sha256(), copied_file) if orig_hash.digest() != copied_hash.digest(): raise tcfl.tc.failed_e("Hashes in copied files changed") self.report_pass("Bigger file copied around is identical")
def eval_06_copy_to(self, target): target.ssh.copy_to(__file__) base_file = os.path.basename(__file__) copied_file = os.path.join(self.tmpdir, base_file) target.ssh.copy_from(base_file, copied_file) orig_hash = commonl.hash_file(hashlib.sha256(), __file__) copied_hash = commonl.hash_file(hashlib.sha256(), copied_file) if orig_hash.digest() != copied_hash.digest(): raise tcfl.tc.failed_e("Hashes in copied files changed") self.report_pass("Bigger file copied around is identical")
def eval_06_copy_to(self, target): # test copying file relative to the script source target.ssh.copy_to('data/beep.wav') base_file = os.path.basename(__file__) # this file target.ssh.copy_to(base_file) copied_file = os.path.join(self.tmpdir, base_file) target.ssh.copy_from(base_file, copied_file) orig_hash = commonl.hash_file(hashlib.sha256(), __file__) copied_hash = commonl.hash_file(hashlib.sha256(), copied_file) if orig_hash.digest() != copied_hash.digest(): raise tcfl.tc.failed_e("Hashes in copied files changed") self.report_pass("Bigger file copied around is identical")
def rest_tb_target_images_upload(rtb, _images): # COMPAT """ Upload images from a list images :param dict rtb: Remote Target Broker :param _images: list of images, which can be specified as: - string with ``"IMAGE1:FILE1 IMAGE2:FILE2..."`` - list or set of strings ``["IMAGE1:FILE1", "IMAGE2:FILE2", ...]`` - list or set of tuples ``[("IMAGE1", "FILE1"), ("IMAGE2", "FILE2"), ...]`` :returns: list of remote images (that can be fed straight to :meth:`tcfl.ttb_client.rest_target_broker.rest_tb_target_images_set`) """ images = [] if isinstance(_images, basestring): for image_spec in _images: try: t, f = image_spec.split(":", 1) images.append((t, f)) except ValueError as _e: raise ValueError("Bad image specification `%s` " "(expecting TYPE:FILE)" % image_spec) elif isinstance(_images, set) or isinstance(_images, list): for image_spec in _images: if isinstance(image_spec, basestring): t, f = image_spec.split(":", 1) images.append((t, f)) elif isinstance(image_spec, tuple) and len(image_spec) == 2: images.append(image_spec) else: raise TypeError("Invalid image specification %s" % image_spec) else: raise TypeError("_images is type %s" % type(_images).__name__) remote_images = {} for image_type, local_filename in images: logger.info("%s: uploading %s", rtb._url, local_filename) digest = commonl.hash_file(hashlib.sha256(), local_filename)\ .hexdigest()[:10] remote_filename = commonl.file_name_make_safe( os.path.abspath(local_filename)) + "-" + digest rtb.rest_tb_file_upload(remote_filename, local_filename) remote_images[image_type] = remote_filename return remote_images
def test_broker_file_upload__simple(self): filename = self.ttbd_config.name sp = commonl.subpython( self.srcdir + "/tcf --config-path : -vvv --url %s broker-file-upload %s dest/file %s" % (self.url, self.target_names[0], filename)) r = sp.join() self.assertEqual( r, 0, msg = sp.output_str + type(self).ttbd_info()) # Check the file made it to the daemon's file storage (it's in # localhost, so we know it is there) expected_filename = os.path.normpath("%s/local/dest/file" % ( self.ttbd_files_dir)) logging.debug("expected_filename %s", expected_filename) if not os.path.isfile(expected_filename): self.fail("expected file '%s' not found\n" % expected_filename \ + sp.output_str + type(self).ttbd_info()) if not filecmp.cmp(expected_filename, filename): self.fail("uploaded file and original file are different") # Check we can list it and it matches the hash sp = commonl.subpython( self.srcdir + "/tcf --config-path : --url %s broker-file-list %s" \ % (self.url, self.target_names[0])) r = sp.join() self.assertEqual( r, 0, msg = sp.output_str + type(self).ttbd_info()) for line in sp.stdout_lines: listed_filename, hexdigest = line.split(" ", 1) if '/dest/file' == listed_filename: self.assertEqual( hexdigest, commonl.hash_file(hashlib.sha256(), self.ttbd_config.name).hexdigest())
def config_file_write(self, name, data, bsp = None): """\ Write an extra config file called *NAME*.conf in the Zephyr's App build directory. Note this takes care to only write it if the data is new or the file is unexistant, to avoid unnecesary rebuilds. :param str name: Name for the configuration file; this has to be a valid filename; *.conf* will be added by the function. :param str data: Data to include in the configuration file; this is (currently) valid kconfig data, which are lines of text with # acting as comment character; for example:: CONFIG_UART_CONSOLE_ON_DEV_NAME="UART_1" :param str bsp: (optional) BSP on which to operate; when the target is configured for a :term:`BSP model` which contains multiple Zephyr BSPs, you will need to specify which one to modify. This parameter can be omitted if only one BSP is available in the current BSP Model. *Example* >>> if something: >>> target.zephyr.conf_file_write("mytweaks", >>> 'CONFIG_SOMEVAR=1\\n' >>> 'CONFIG_ANOTHER="VALUE"\\n') """ target = self.target bsp = self._bsp_select(target, bsp) # Ensure the config directory is there outdir = target._kws_bsp[bsp]['zephyr_objdir'] try: os.makedirs(outdir) except OSError as e: if e.errno != errno.EEXIST: raise RuntimeError("%s: Cannot create outdir directory: %s" % (outdir, str(e))) # Now create a .new file if not name.endswith(".conf"): name += ".conf" existing_filename = os.path.join(outdir, name) new_filename = existing_filename + ".new" with codecs.open(new_filename, "w+", encoding = 'utf-8', errors = 'ignore') as f: f.write("""\ # Config file automatically generated by TCF's: # # %s # # because of instructions from # # %s # # Do not edit by hand """ % (commonl.origin_get(), commonl.origin_get(2))) f.write(data) # report the config file we wrote, so reproduction # instructions will carry it in the report file; we report # the data without the header to make it easier -- note we # use report_pass() [vs report_info()] as _info might get # filtered as too verbose info, where as pass is # information important for passing. target.report_pass("enabled Zephyr config file %s at %s" % (name, outdir), { 'config file': data }) if not os.path.exists(existing_filename): shutil.move(new_filename, existing_filename) else: # Check if there are changes before updating it, to avoid # unnecesary rebuilds _new_hash = hashlib.sha256() new_hash = commonl.hash_file(_new_hash, new_filename) _old_hash = hashlib.sha256() old_hash = commonl.hash_file(_old_hash, existing_filename) if new_hash.digest() != old_hash.digest(): shutil.move(new_filename, existing_filename) else: os.unlink(new_filename)
def flash(self, images, upload=True): """Flash images onto target >>> target.images.flash({ >>> "kernel-86": "/tmp/file.bin", >>> "kernel-arc": "/tmp/file2.bin" >>> }, upload = True) or: >>> target.images.flash({ >>> "vmlinuz": "/tmp/vmlinuz", >>> "initrd": "/tmp/initrd" >>> }, upload = True) If *upload* is set to true, this function will first upload the images to the server and then flash them. :param dict images: dictionary keyed by (str) image type of things to flash in the target. e.g.: The types if images supported are determined by the target's configuration and can be reported with :meth:`list` (or command line *tcf images-list TARGETNAME*). :param bool upload: (optional) the image names are local files that need to be uploaded first to the server (this function will take care of that). """ if isinstance(images, dict): for k, v in images.items(): assert isinstance(k, basestring) \ and isinstance(v, basestring), \ "images has to be a dictionary IMAGETYPE:IMAGEFILE;" \ " all strings; %s, %s (%s, %s)" \ % (k, v, type(k), type(v)) else: raise AssertionError( "images has to be a dictionary IMAGETYPE:IMAGEFILE; got %s" \ % type(images)) if self.compat: raise RuntimeError("target does not support new images" " interface, use set() or upload_set()") target = self.target images_str = " ".join("%s:%s" % (k, v) for k, v in images.items()) # if we have to upload them, then we'll transform the names to # point to the names we got when uploading if upload: # Ok, we need to upload--the names in the dictionary point # to local filenames relative to the dir where we are # from, or absolute. Upload them to the server file space # for the user and give them a local name in there. _images = {} target.report_info("uploading: " + images_str, dlevel=2) for img_type, img_name in images.iteritems(): # the remote name will be NAME-DIGEST, so if multiple # testcases for the same user are uploading files with # the same name but different context, they don't # collide digest = commonl.hash_file(hashlib.sha256(), img_name) img_name_remote = \ commonl.file_name_make_safe(os.path.abspath(img_name)) \ + "-" + digest.hexdigest()[:10] target.rtb.rest_tb_file_upload(img_name_remote, img_name) _images[img_type] = img_name_remote target.report_info("uploaded: " + images_str, dlevel=1) else: _images = images # We don't do retries here, we leave it to the server target.report_info("flashing: " + images_str, dlevel=2) target.ttbd_iface_call("images", "flash", images=_images) target.report_info("flashed:" + images_str, dlevel=1)
def config_file_write(self, name, data, bsp = None): """\ Write an extra config file called *NAME*.conf in the Zephyr's App build directory. Note this takes care to only write it if the data is new or the file is unexistant, to avoid unnecesary rebuilds. :param str name: Name for the configuration file; this has to be a valid filename; *.conf* will be added by the function. :param str data: Data to include in the configuration file; this is (currently) valid kconfig data, which are lines of text with # acting as comment character; for example:: CONFIG_UART_CONSOLE_ON_DEV_NAME="UART_1" :param str bsp: (optional) BSP on which to operate; when the target is configured for a :term:`BSP model` which contains multiple Zephyr BSPs, you will need to specify which one to modify. This parameter can be omitted if only one BSP is available in the current BSP Model. *Example* >>> if something: >>> target.zephyr.conf_file_write("mytweaks", >>> 'CONFIG_SOMEVAR=1\\n' >>> 'CONFIG_ANOTHER="VALUE"\\n') """ target = self.target bsp = self._bsp_select(target, bsp) # Ensure the config directory is there outdir = target._kws_bsp[bsp]['zephyr_objdir'] try: os.makedirs(outdir) except OSError as e: if e.errno != errno.EEXIST: raise RuntimeError("%s: Cannot create outdir directory: %s" % (outdir, e.message)) # Now create a .new file if not name.endswith(".conf"): name += ".conf" existing_filename = os.path.join(outdir, name) new_filename = existing_filename + ".new" with codecs.open(new_filename, "w+", encoding = 'utf-8', errors = 'ignore') as f: f.write("""\ # Config file automatically generated by TCF's: # # %s # # because of instructions from # # %s # # Do not edit by hand """ % (commonl.origin_get(), commonl.origin_get(2))) f.write(data) # report the config file we wrote, so reproduction # instructions will carry it in the report file; we report # the data without the header to make it easier -- note we # use report_pass() [vs report_info()] as _info might get # filtered as too verbose info, where as pass is # information important for passing. target.report_pass("enabled Zephyr config file %s at %s" % (name, outdir), { 'config file': data }) if not os.path.exists(existing_filename): shutil.move(new_filename, existing_filename) else: # Check if there are changes before updating it, to avoid # unnecesary rebuilds _new_hash = hashlib.sha256() new_hash = commonl.hash_file(_new_hash, new_filename) _old_hash = hashlib.sha256() old_hash = commonl.hash_file(_old_hash, existing_filename) if new_hash.digest() != old_hash.digest(): shutil.move(new_filename, existing_filename) else: os.unlink(new_filename)