def _handlePowerData(self, filename): if not os.path.isfile(filename): getLogger().error("Power data file " "{} doesn't exist".format(filename)) return app = UploadDownloadFiles(self.args) file_link = app.upload(file=filename, permanent=False) # remove the temporary file os.remove(filename) return file_link
def __init__(self, args): self.config_filename = args.cache_config self.root_dir = args.root_model_dir self.file_storage = UploadDownloadFiles(args) self.config = None if not os.path.isfile(self.config_filename): self.config = {} else: with open(self.config_filename, 'r') as f: try: self.config = json.load(f) except Exception: self.config = {} self._updateConfig()
class FileHandler(object): def __init__(self, args): self.config_filename = args.cache_config self.root_dir = args.root_model_dir self.file_storage = UploadDownloadFiles(args) self.config = None if not os.path.isfile(self.config_filename): self.config = {} else: with open(self.config_filename, 'r') as f: try: self.config = json.load(f) except Exception: self.config = {} self._updateConfig() def uploadFile(self, filename, md5, basefilename, cache_file): if filename.startswith("https://") or filename.startswith("http://"): return filename, md5 if filename.startswith("specifications"): """ We will handle the spcical case here that the file is from internal binary. We will first load it, save it as a temp file, and then return the temp path. In general, we don't encourage this case. """ if not pkg_resources.resource_exists("aibench", filename): getLogger().error("Cannot find {}".format(filename)) raw_context = pkg_resources.resource_string("aibench", filename) temp_name = filename.split("/")[-1] temp_dir = tempfile.mkdtemp() path = os.path.join(temp_dir, temp_name) with open(path, "w") as f: f.write(raw_context.decode("utf-8")) elif filename.startswith("//"): assert self.root_dir, \ "root_dir must be specified for relative path" path = self.root_dir + filename[1:] elif filename.startswith("/"): path = filename else: path = os.path.dirname( os.path.realpath(basefilename)) + "/" + filename if not os.path.isfile(path): getLogger().info("Skip uploading {}".format(filename)) return filename, md5 upload_path, cached_md5 = self._getCachedFile(path) base_filename = os.path.basename(filename) if upload_path is None or not cache_file or md5 is not cached_md5: upload_path = self.file_storage.upload(orig_path=filename, file=path, permanent=False) if cache_file or md5 is not cached_md5: md5 = self._saveCachedFile(path, upload_path) else: getLogger().info( "File {} cached, skip uploading".format(base_filename)) return upload_path, md5 def _getCachedFile(self, path): md5 = None if path in self.config: entry = self.config[path] if os.path.isfile(path): md5 = self._calculateMD5(path) if entry["md5"] == md5: return entry["upload_path"], md5 return None, md5 def _saveCachedFile(self, path, upload_path): calculate_md5 = self._calculateMD5(path) entry = { "local_path": path, "md5": calculate_md5, "time": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "upload_path": upload_path } self.config[path] = entry self._updateConfigFile() return calculate_md5 def _calculateMD5(self, filename): m = hashlib.md5() m.update(open(filename, 'rb').read()) md5 = m.hexdigest() return md5 def _updateConfigFile(self): json_file = json.dumps(self.config, indent=2, sort_keys=True) with open(self.config_filename, 'w') as f: f.write(json_file) def _updateConfig(self): cutoff_time = datetime.datetime.now() - datetime.timedelta(days=21) # Delete files that are three weeks old updated = False keys = list(self.config.keys()) for path in keys: entry = self.config[path] t = datetime.datetime.strptime(entry["time"], "%Y-%m-%d %H:%M:%S") if t < cutoff_time: updated = True del self.config[path] if updated: self._updateConfigFile()