Ejemplo n.º 1
0
    def _proc(self, id_asset, db):
        asset = Asset(id_asset, db = db)
        for id_action in self.conditions:
            if "broker/started/{}".format(id_action) in asset.meta:
                continue
            cond_title, cond, priority = self.conditions[id_action]
            if eval(cond):
                logging.info("{} matches action condition {}".format(asset, cond_title))
                res, msg = send_to(asset.id, id_action, settings={}, id_user=0, priority=priority, restart_existing=False, db=db)

                if success(res):
                    logging.info(msg)
                else:
                    logging.error(msg)

                asset["broker/started/{}".format(id_action)] = 1
                asset.save()
Ejemplo n.º 2
0
    def _proc(self, id_asset, db):
        asset = Asset(id_asset, db=db)
        for analyzer in self.analyzers:

            qinfo = asset["qc/analyses"] or {}
            if type(qinfo) in [str, unicode]:
                qinfo = json.loads(qinfo)

            if analyzer.proc_name in qinfo and (
                    qinfo[analyzer.proc_name] == -1
                    or qinfo[analyzer.proc_name] >= analyzer.version):
                continue

            if eval(analyzer.condition):
                logging.info("Analyzing {} using '{}'".format(
                    asset, analyzer.proc_name))
                a = analyzer(asset)

                #
                # Reload asset (it may be changed by someone during analysis
                #

                del (asset)
                asset = Asset(id_asset, db=db)
                result = -1 if not a.status else analyzer.version

                qinfo = asset["qc/analyses"] or {}
                if type(qinfo) in [str, unicode]:
                    qinfo = json.loads(qinfo)
                qinfo[analyzer.proc_name] = result
                asset["qc/analyses"] = qinfo

                #
                # Save result
                #

                for key in a.result:
                    value = a.result[key]
                    if value:
                        logging.debug("Set {} {} to {}".format(
                            asset, key, value))
                        asset[key] = value
                asset.save()
                self.heartbeat()
Ejemplo n.º 3
0
    def finalize(self):
        new = None
        asset = Asset(
            self.asset.id)  # Reload asset (possibly changed during encoding)

        if self.task.find("target").text == "new":
            id_storage = self.id_storage
            r = asset_by_path(id_storage, self.target_rel_path)
            if r:
                new = Asset(r)
                logging.info("Updating asset {!r}".format(new))
                keys = new.meta.keys()
                for key in keys:
                    if key in meta_types and meta_types[key].namespace in [
                            "qc", "fmt"
                    ]:
                        new[key] = ""
            else:
                logging.info(
                    "Creating new asset for {!r} conversion.".format(asset))
                new = Asset()
                new["media_type"] = FILE
                new["content_type"] = VIDEO

                new["version_of"] = asset.id
                new["id_storage"] = id_storage
                new["path"] = self.target_rel_path
                new["origin"] = "Video conversion"
                new["id_folder"] = asset["id_folder"]

                for key in asset.meta:
                    if key in meta_types and meta_types[key].namespace in [
                            "AIEB", "m"
                    ]:
                        new[key] = asset[key]

            new["status"] = CREATING

        for intra in self.task.findall("intra"):
            exec(intra.text)

        try:
            os.rename(self.temp_file_path, self.target_file_path)
        except:
            return "Unable to move output file to target destination"

        if new is not None:
            new.save()

        for post in self.task.findall("post"):
            exec(post.text)

        if new is not None:
            new.save()
        asset.save()
Ejemplo n.º 4
0
    def _proc(self, id_asset, db):
        asset = Asset(id_asset, db=db)
        fname = asset.file_path

        if asset["id_storage"] not in self.mounted_storages:
            return

        if not os.path.exists(fname):
            if asset["status"] in [ONLINE, RESET, CREATING]:
                logging.warning(
                    "Turning offline {} (File does not exist)".format(asset))
                asset["status"] = OFFLINE
                asset.save()
            return

        try:
            fmtime = int(os.path.getmtime(fname))
            fsize = int(os.path.getsize(fname))
        except:
            log_traceback("Unable to get file attrs {}".format(asset))
            return

        if fsize == 0:
            if asset["status"] != OFFLINE:
                logging.warning(
                    "Turning offline {} (empty file)".format(asset))
                asset["status"] = OFFLINE
                asset.save()
            return

        if fmtime != asset["file/mtime"] or asset["status"] == RESET:
            try:
                f = open(fname, "rb")
            except:
                logging.debug("{} creation in progress.".format(asset))
                return
            else:
                f.seek(0, 2)
                fsize = f.tell()
                f.close()

            if asset["status"] == RESET:
                asset.load_sidecar_metadata()

            # Filesize must be changed to update metadata automatically.
            # It sucks, but mtime only condition is.... errr doesn't work always

            if fsize == asset["file/size"] and asset["status"] != RESET:
                logging.debug(
                    "{} file mtime has been changed. Updating.".format(asset))
                asset["file/mtime"] = fmtime
                asset.save(set_mtime=False, notify=False)
            else:
                logging.info("Updating {}".format(asset))

                keys = list(asset.meta.keys())
                for key in keys:
                    if meta_types[key].namespace in ("fmt", "qc"):
                        del (asset.meta[key])

                asset["file/size"] = fsize
                asset["file/mtime"] = fmtime

                #########################################
                ## PROBE

                for probe in probes:
                    if probe.accepts(asset):
                        logging.debug("Probing {} using {}".format(
                            asset, probe))
                        asset = probe.work(asset)

                ## PROBE
                #########################################

                if asset["status"] == RESET:
                    asset["status"] = ONLINE
                    logging.info("{} reset completed".format(asset))
                else:
                    asset["status"] = CREATING
                asset.save()

        if asset["status"] == CREATING and asset["mtime"] + 15 > time.time():
            logging.debug("Waiting for {} completion assurance.".format(asset))
            asset.save(set_mtime=False, notify=False)

        elif asset["status"] in (CREATING, OFFLINE):
            logging.goodnews("Turning online {}".format(asset))
            asset["status"] = ONLINE
            asset.save()

            db = DB()
            db.query(
                """UPDATE nx_jobs SET progress=-1, id_service=0, ctime=%s, stime=0, etime=0, id_user=0, message='Restarting after source update'
                    WHERE id_object=%s AND id_action > 0 and progress IN (-2, -3)""",
                [time.time(), id_asset])

            db.commit()
Ejemplo n.º 5
0
    def on_main(self):
        db = DB()
        self.existing = []
        start_time = time.time()
        db.query("SELECT meta FROM assets WHERE media_type=1 AND status=1")
        for (meta, ) in db.fetchall():
            asset = Asset(meta=meta, db=db)
            file_path = asset.file_path
            self.existing.append(file_path)
        duration = time.time() - start_time
        if duration > 5 or config.get("debug_mode", False):
            logging.debug(f"Online assets loaded in {s2time(duration)}")

        start_time = time.time()
        for wf_settings in self.settings.findall("folder"):
            id_storage = int(wf_settings.attrib["id_storage"])
            rel_wf_path = wf_settings.attrib["path"]
            quarantine_time = int(
                wf_settings.attrib.get("quarantine_time", "10"))
            id_folder = int(wf_settings.attrib.get("id_folder", 12))

            storage_path = storages[id_storage].local_path
            watchfolder_path = os.path.join(storage_path, rel_wf_path)

            if not os.path.exists(watchfolder_path):
                logging.warning("Skipping non-existing watchfolder",
                                watchfolder_path)
                continue

            i = 0
            for file_object in get_files(
                    watchfolder_path,
                    recursive=wf_settings.attrib.get("recursive", False),
                    hidden=wf_settings.attrib.get("hidden", False),
                    case_sensitive_exts=wf_settings.get(
                        "case_sensitive_exts", False),
            ):
                i += 1
                if i % 100 == 0 and config.get("debug_mode", False):
                    logging.debug("{} files scanned".format(i))

                if not file_object.size:
                    continue

                full_path = file_object.path
                if full_path in self.existing:
                    continue

                now = time.time()
                asset_path = full_path.replace(storage_path, "", 1).lstrip("/")
                ext = os.path.splitext(asset_path)[1].lstrip(".").lower()
                if ext not in FileTypes.exts():
                    continue

                asset = asset_by_path(id_storage, asset_path, db=db)
                if asset:
                    self.existing.append(full_path)
                    continue

                base_name = get_base_name(asset_path)

                if quarantine_time and now - file_object.mtime < quarantine_time:
                    logging.debug(f"{base_name} is too young. Skipping")
                    continue

                asset = Asset(db=db)
                asset["content_type"] = FileTypes.by_ext(ext)
                asset["media_type"] = MediaType.FILE
                asset["id_storage"] = id_storage
                asset["path"] = asset_path
                asset["ctime"] = now
                asset["mtime"] = now
                asset["status"] = ObjectStatus.CREATING
                asset["id_folder"] = id_folder
                asset["title"] = base_name

                asset.load_sidecar_metadata()

                failed = False
                for post_script in wf_settings.findall("post"):
                    try:
                        exec(post_script.text)
                    except Exception:
                        log_traceback(
                            f"Error executing post-script on {asset}")
                        failed = True

                if not failed:
                    asset.save(set_mtime=False)

        duration = time.time() - start_time
        if duration > 60 or config.get("debug_mode", False):
            logging.debug(f"Watchfolders scanned in {s2time(duration)}")