def add_path(self,name, path):
if name=='default': self._base = path
self.paths[name] = path
if self.instant_reload:
observer = Observer()
observer.schedule(self.reload_handler, path=path, recursive=True)
observer.start()
def code():
# file monitor server
observer = Observer()
# py yml file monitor
patterns = ["*.py", "*demo.yml"] # '*' is necessary, and must in the first.
restart_processor = ServerStarter(
[
{"cmd": "rm -rf %s/*.log" % os.path.join(workspace, "log"), "is_daemon": False},
{"cmd": "./run.py run", "network_port": (config["simple_server"]["port"],)},
]
)
monitor = SourceCodeMonitor(restart_processor, patterns)
observer.schedule(monitor, program_dir, recursive=True)
observer.schedule(monitor, http_api.__path__[0], recursive=True)
# # rebuild css and js's min file while source file is change
# patterns = ['*.css', '*.js', '*static.yml'] # '*' is necessary, and must in the first.
# monitor = SourceCodeMonitor(BuildCssJsProcessor(program_dir, static), patterns, None, 500)
# observer.schedule(monitor, program_dir, recursive=True)
# start monitoring
observer.start()
try:
time.sleep(31536000) # one year
except KeyboardInterrupt:
observer.stop()
def main():
if (len(sys.argv) > 1 and os.path.exists(sys.argv[1]) and os.path.isfile(sys.argv[1])):
filename = sys.argv[1]
else:
filename = gui.get_path("*.csv",defaultFile="data.csv")
commonPath = os.path.abspath(os.path.split(filename)[0])
outputFile = os.path.join(commonPath, "odmanalysis.csv")
print "Now watching %s for changes" % filename
handler = OMDCsvChunkHandler(filename,outputFile)
observer = Observer()
observer.schedule(handler, path=commonPath, recursive=False)
handler.startPCChain()
observer.start()
try:
while True:
time.sleep(1)
except (KeyboardInterrupt, SystemExit):
print "Stopping..."
observer.stop()
time.sleep(1)
observer.join()
def main():
arguments = docopt(__doc__, version='Storyline HTTP v0.1')
if arguments.get('--debug'):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stderr)
logger.addHandler(handler)
app.config.from_object(__name__)
app.debug = arguments.get('--debug')
story_path = arguments.get('STORY_PATH', '.')
global plot
plot = storyfile.load_plot_from_path(story_path)
observer = Observer()
observer.schedule(LoggingEventHandler(), path=story_path, recursive=True)
observer.schedule(Reloader(story_path), path=story_path, recursive=True)
observer.start()
try:
app.run()
finally:
observer.stop()
observer.join()
def watch(self, source, write=True, package=None, run=False, force=False):
"""Watches a source and recompiles on change."""
from watchdog.events import FileSystemEventHandler
from watchdog.observers import Observer
def recompile(path):
if os.path.isfile(path) and os.path.splitext(path)[1] in code_exts:
self.compile_path(path, write, package, run, force)
class watcher(FileSystemEventHandler):
def on_modified(_, event):
recompile(event.src_path)
def on_created(_, event):
recompile(event.src_path)
source = fixpath(source)
self.console.show("Watching "+showpath(source)+" ...")
self.console.print("(press Ctrl-C to end)")
observer = Observer()
observer.schedule(watcher(), source, recursive=True)
observer.start()
try:
while True:
time.sleep(.1)
except KeyboardInterrupt:
pass
finally:
observer.stop()
observer.join()
def main():
global badExtensionCounter, failedFlag, pool, failedProcessCounter#, db
sql_setup() # Set-up SQL Database/check to see if exists
# Initiate File Path Handler
observer = Observer()
observer.schedule(MyHandler(), path=file_path, recursive=True)
observer.start()
cpuCount = multiprocessing.cpu_count() # Count all available CPU's
print "\nTotal CPU Count: %d"%(cpuCount)
pool = multiprocessing.Pool(4, worker,(processQueue,)) # Create 4 child processes to handle all queued elements
active = multiprocessing.active_children() # All active child processes
print "Total number of active child processes: %s\n"%(str(active))
try:
while True:
time.sleep(0.2)
except KeyboardInterrupt:
pool.terminate() # Stop all child processes
pool.join() # Join the processes with parent and terminate
active = multiprocessing.active_children() # All active child processes, list should be empty at this point.
print "\nTotal number of active child processes: %s\n"%(str(active))
shutdown() # Run shutdown sequence
observer.stop()
observer.join()
sys.exit(1)
def generate_and_observe(args, event):
while event.isSet():
# Generate the presentation
monitor_list = generate(args)
print("Presentation generated.")
# Make a list of involved directories
directories = defaultdict(list)
for file in monitor_list:
directory, filename = os.path.split(file)
directories[directory].append(filename)
observer = Observer()
handler = HovercraftEventHandler(monitor_list)
for directory, files in directories.items():
observer.schedule(handler, directory, recursive=False)
observer.start()
while event.wait(1):
time.sleep(0.05)
if handler.quit:
break
observer.stop()
observer.join()
class WatchFile(object):
def __init__(self, send_msg_func, *args, **kargs):
self.path = kargs['path'] if kargs.has_key('path') else '.'
self.suffix = kargs['suffix'] if kargs.has_key('suffix') else '*' # star represent any file
self.observer = Observer()
self.event_handler = MyFileMonitor(self.suffix, callback=self.get_data)
self.send_msg_func = send_msg_func
self.filename = self.zip_filename = ''
def run(self):
self.observer.schedule(self.event_handler, self.path, recursive=True)
self.observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
self.observer.stop()
self.observer.join()
def get_data(self, filename):
data = self._unpack(filename)
data = str(data)
print(data, type(data))
self.send_msg_func(data)
def _unpack(self, filename):
# first rename suffix to zip file
# may not work on linux
if system == 'Windows':
filename = filename[2:] if filename.startswith('.\\') else filename
filename = filename.lstrip()
new_name = filename.split('.')[0] + '.zip'
new_name = new_name[1:] if new_name.startswith('\\') else new_name
elif system == 'Linux':
new_name = filename
print('Old name:', filename, ' New name:', new_name)
self.filename = filename
self.zip_filename = new_name
# waiting for operating sys create the file
time.sleep(3)
os.rename(filename, new_name)
zip_file = zipfile.ZipFile(new_name, 'r')
json_data = ""
for name in zip_file.namelist():
if name == "project.json":
file = zip_file.open(name, 'r')
json_data = "".join(file.readlines())
# change filename back to .sb2
if new_name.endswith('.zip'):
os.rename(new_name, filename)
return self.get_cmd(json_data)
def get_cmd(self, json_data):
jsonfy_data = json.loads(json_data)
child = jsonfy_data['children'][0]
scripts = child['scripts']
return scripts
class FSWatcher(object):
def __init__(self):
self.observer = Observer()
for path in odoo.modules.module.ad_paths:
_logger.info('Watching addons folder %s', path)
self.observer.schedule(self, path, recursive=True)
def dispatch(self, event):
if isinstance(event, (FileCreatedEvent, FileModifiedEvent, FileMovedEvent)):
if not event.is_directory:
path = getattr(event, 'dest_path', event.src_path)
if path.endswith('.py'):
try:
source = open(path, 'rb').read() + '\n'
compile(source, path, 'exec')
except SyntaxError:
_logger.error('autoreload: python code change detected, SyntaxError in %s', path)
else:
_logger.info('autoreload: python code updated, autoreload activated')
restart()
def start(self):
self.observer.start()
_logger.info('AutoReload watcher running')
def stop(self):
self.observer.stop()
self.observer.join()
class PropMTimeWatcher:
def __init__(self, app_data_folder):
self._app_data_folder = app_data_folder
self._observer = Observer()
self.schedule()
def schedule(self):
pref = PropMTimePreferences(self._app_data_folder)
self._observer.unschedule_all()
for path, watcher in pref.get_all_paths().items():
if watcher:
if os.path.exists(path):
event_handler = ModHandler(path, self._app_data_folder)
log.info('scheduling watcher : %s' % path)
self._observer.schedule(event_handler, path=path, recursive=True)
else:
log.error('Error: "%s" does not exist.\n\nPlease edit the path.\n\nTo do this, click on the %s icon and select "Paths".' %
(path, __application_name__))
self._observer.start()
def request_exit(self):
self._observer.unschedule_all()
self._observer.stop()
self._observer.join(TIMEOUT)
if self._observer.isAlive():
log.error('observer still alive')
def main():
# Fill all changes that occurred when track-changes.py wasn't running.
if os.path.isdir("out"):
shutil.rmtree("out", True)
if not os.path.isdir("out"):
os.mkdir("out")
startup_changes.sync_offline_changes("posts", "out")
print "Watching posts directory for changes... CTRL+C to quit."
watch_directory = "posts"
event_handler = MyHandler()
# Run the watchdog.
observer = Observer()
observer.schedule(event_handler, watch_directory, True)
observer.start()
"""
Keep the script running or else python closes without stopping the observer
thread and this causes an error.
"""
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def __init__(self, config):
"""
Initialize the watcher, use the config passed from main
"""
self.config = config
# List of pending files
self.pending_files = set()
self.sync_timer = None
# Setup our watchdog observer
observer = Observer()
observer.schedule(ChangeHandler(self.on_file_changed), path=config.directory, recursive=True)
observer.start()
logging.info("Starting change tracker, cmd: {}, dir: {}, delay: {}".format(config.sync_cmd,
config.directory,
config.delay))
try:
while True:
time.sleep(0.5)
except KeyboardInterrupt:
observer.stop()
observer.join()
def stream_video(video_path):
global VIDEO_BITRATE
global AUDIO_BITRATE
create_working_directory()
head, tail = os.path.split(video_path)
name = tail.split('.')[0]
nonce = '-' + ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(5))
if '.avi' in video_path:
args = "-i %s -vcodec h264 -b %s -acodec libfaac -ab %s -f hls ./.kickflip/%s.m3u8"
args = args % (video_path, VIDEO_BITRATE, AUDIO_BITRATE, name+nonce)
else:
args = "-i %s -f hls -codec copy ./.kickflip/%s.m3u8"
args = args % (video_path, name+nonce)
observer = Observer()
observer.schedule(SegmentHandler(), path='./.kickflip')
observer.start()
time.sleep(3) # This is a f*****g hack.
process = envoy.run('ffmpeg ' + args)
observer.stop()
upload_file(video_path)
return ''
def watch(path, handler=None, debug=True):
import time
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
class Handler(FileSystemEventHandler):
def on_any_event(self, event):
if debug:
print "File {0}: {1}".format(event.event_type, event.src_path)
if not handler:
print "No handler specified"
return
handler(event.src_path, event.event_type)
event_handler = Handler()
observer = Observer()
observer.schedule(event_handler, path, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class ActivityCheck(Thread):
def __init__(self, period, path, mailhost, fromaddr, toaddrs):
Thread.__init__(self)
self.period = int(period)
self.path = path
self.activity = False
self.last_time = datetime.datetime.now()
self.message_sent = False
self.subject = 'WARNING : ' + HOSTNAME + ' : ' + 'telecaster monitor activity'
self.logger = EmailLogger(mailhost, fromaddr, toaddrs, self.subject)
self.event_handler = ActivityEventHandler(ignore_patterns=IGNORE_PATTERNS)
self.observer = Observer()
self.observer.schedule(self.event_handler, path, recursive=True)
self.observer.start()
def run(self):
while True:
if not self.event_handler.activity:
now = datetime.datetime.now()
delta = now - self.last_time
if delta.total_seconds() > LOG_MAX_PERIOD or not self.message_sent:
self.logger.logger.error('The monitor is NOT recording anymore in ' + self.path + ' ! ')
self.last_time = now
self.message_sent = True
else:
self.event_handler.activity = False
time.sleep(self.period)
def stop(self):
self.observer.stop()
def main():
pandoc_path = which("pandoc")
if not pandoc_path :
print "pandoc executable must be in the path to be used by pandoc-watch!"
exit()
config = Configuration.Instance()
parseOptions()
config.setDirContentAndTime(getDirectoryWatchedElements())
print "Starting pandoc watcher ..."
while True:
event_handler = ChangeHandler()
observer = Observer()
observer.schedule(event_handler, os.getcwd(), recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt as err:
print str(err)
observer.stop()
print "Stopping pandoc watcher ..."
exit()
def _watch(root=u'.', dest=u'_site', pattern=u'**/*.html', exclude=u'_*/**'):
try:
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
except ImportError:
msg = "The build --watch feature requires watchdog. \n"\
+ "Please install it with 'easy_install watchdog'."
print(msg)
return None
class handler(FileSystemEventHandler):
def on_any_event(self, event):
exclude_path = os.path.join(os.getcwd(), exclude)
if not utils.matches_pattern(exclude_path, event.src_path):
build_files(root=root,
dest=dest,
pattern=pattern,
exclude=exclude)
observer = Observer()
observer.schedule(handler(), root, recursive=True)
observer.start()
print("Watching '{0}' ...".format(root))
return observer
def __init__(self, zkconn, root_node_path, conf):
super(ZkFarmJoiner, self).__init__()
self.update_remote_timer = None
self.update_local_timer = None
self.zkconn = zkconn
self.conf = conf
self.node_path = "%s/%s" % (root_node_path, self.myip())
# force the hostname info key
info = conf.read()
info["hostname"] = gethostname()
conf.write(info)
zkconn.create(self.node_path, serialize(conf.read()), zc.zk.OPEN_ACL_UNSAFE, EPHEMERAL)
observer = Observer()
observer.schedule(self, path=conf.file_path, recursive=True)
observer.start()
zkconn.get(self.node_path, self.node_watcher)
while True:
with self.cv:
self.wait()
def watch(directory=None, auto_clear=False, beep_on_failure=True,
onpass=None, onfail=None, extensions=[]):
"""
Starts a server to render the specified file or directory
containing a README.
"""
if directory and not os.path.isdir(directory):
raise ValueError('Directory not found: ' + directory)
directory = os.path.abspath(directory or '')
# Initial run
event_handler = ChangeHandler(directory, auto_clear, beep_on_failure,
onpass, onfail, extensions)
event_handler.run()
# Setup watchdog
observer = Observer()
observer.schedule(event_handler, path=directory, recursive=True)
observer.start()
# Watch and run tests until interrupted by user
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def watch_project(markdown_fn, output_fn, template_fn, render_first=True):
class Handler(FileSystemEventHandler):
def on_any_event(self, event):
if event.src_path == os.path.abspath(output_fn):
return
print('Rendering slides...')
process_slides(markdown_fn, output_fn, template_fn)
if render_first == True:
process_slides(markdown_fn, output_fn, template_fn)
observer = Observer()
event_handler = Handler()
dirname = os.path.dirname(os.path.abspath(markdown_fn))
observer.schedule(event_handler, path=dirname, recursive=True)
print("Watching for events on {:s}...".format(dirname))
observer.start()
try:
while True:
time.sleep(0.5)
except KeyboardInterrupt:
observer.stop()
observer.join()
class AlertEventHandler(FileSystemEventHandler):
def __init__(self, sensor_uuid, alert_file, hpc):
"""
Initializes a filesytem watcher that will watch
the specified file for changes.
`alert_file` is the absolute path of the snort alert file.
`hpc` is the hpfeeds client
"""
self.sensor_uuid = sensor_uuid
self.alert_file = alert_file
self.hpc = hpc
logger.info('connected to hpfeeds broker {}'.format(hpc.brokername))
self.observer = Observer()
self.observer.schedule(self, os.path.dirname(alert_file), False)
@property
def latest_alert_date(self):
return safe_unpickle('alert_date.pkl')
@latest_alert_date.setter
def latest_alert_date(self, date):
safe_pickle('alert_date.pkl', date)
def on_any_event(self, event):
if (not event.event_type == 'deleted') and (event.src_path == self.alert_file):
alerts = Alert.from_log(self.sensor_uuid, self.alert_file, self.latest_alert_date)
if alerts:
logger.info("submitting {} alerts to {}".format(len(alerts), self.hpc.brokername))
alerts.sort(key=lambda e: e.date)
self.latest_alert_date = alerts[-1].date
for alert in alerts:
self.hpc.publish("snort.alerts", alert.to_json())
def main():
if not config['play']['scan']:
raise Exception('''
Nothing to scan. Add a path in the config file.
Example:
play:
scan:
-
type: shows
path: /a/path/to/the/shows
''')
obs = Observer()
for s in config['play']['scan']:
event_handler = Handler(
scan_path=s['path'],
type_=s['type'],
)
obs.schedule(
event_handler,
s['path'],
recursive=True,
)
obs.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
obs.stop()
obs.join()
def __init__(self, input_dir, templates_dir):
paths = [input_dir, templates_dir]
threads = []
try:
observer = Observer()
event_handler = WatchEventHandler()
for i in paths:
targetPath = str(i)
observer.schedule(event_handler, targetPath, recursive=True)
threads.append(observer)
observer.start()
signal_watch_init = signal('watch_init')
signal_watch_init.send(self)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
wrangler._reporter.log("Stopping with grace and poise", "green")
observer.stop()
observer.join()
except:
return None
class Modsync:
_target = []
_source = ''
_observer = None
def __init__(self):
pass
def setSource(self, source):
self._source = source
def setTarget(self, target_dir):
self._target.append(target_dir)
def getObserver(self):
return self._observer
def run(self):
if not self._source:
return 0
self._observer = Observer()
event_handler = ModsyncEventHandler(self._observer, self._source, self._target)
self._observer.schedule(event_handler, self._source, recursive=True)
self._observer.start()
try:
time.sleep(2)
pass
except KeyboardInterrupt:
self._observer.stop()
self._observer.join()
return 0
class Sceduler:
def __init__(self, config):
fs = config.get('scheduler', 'fs', 0)
dest = config.get('store', 'path', 0)
self.ioqueue = Queue()
self.iothread = Thread(target=self.ioprocess)
self.iothread.daemon = True
self.observer = Observer()
self.event_handler = IoTask(self.ioqueue, fs, dest)
self.observer.schedule(self.event_handler, fs, recursive=True)
def ioprocess(self):
while True:
t = self.ioqueue.get()
try:
t.process()
finally:
self.ioqueue.task_done()
def start(self):
self.observer.start()
self.iothread.start()
def stop(self):
self.observer.stop()
self.iothread.stop()
def join(self):
self.observer.join()
self.iothread.join()
def start(filename,format,callback=None,verbose=False): observer = Observer() fm = FileMonitor(observer,filename,format,callback,verbose) fm._handle() observer.schedule(fm, path=os.path.dirname(filename), recursive=False) observer.start() return fm
class RoleBasedAuthorizationProvider(AbstractAuthorizationProvider,
FileSystemEventHandler):
def __init__(self, role_loader, roles_config_file_path):
self.lgr = logging.getLogger(FLASK_SECUREST_LOGGER_NAME)
self.role_loader = role_loader
self.permissions_by_roles = None
self.roles_config_file_path = os.path.abspath(roles_config_file_path)
self.observer = Observer()
self.observer.schedule(self,
path=os.path.dirname(
self.roles_config_file_path),
recursive=False)
self.load_roles_config()
self.observer.start()
def load_roles_config(self):
try:
with open(self.roles_config_file_path, 'r') as config_file:
self.permissions_by_roles = yaml.safe_load(config_file.read())
self.lgr.info('Loading of roles configuration ended '
'successfully')
except (yaml.parser.ParserError, IOError) as e:
err = 'Failed parsing {role_config_file} file. Error: {error}.' \
.format(role_config_file=self.roles_config_file_path, error=e)
self.lgr.warning(err)
raise ValueError(err)
def on_modified(self, event):
if os.path.abspath(event.src_path) == self.roles_config_file_path:
self.load_roles_config()
def authorize(self):
target_endpoint = rest_security.get_endpoint()
target_method = rest_security.get_http_method()
roles = self.role_loader.get_roles()
return self._is_allowed(target_endpoint, target_method, roles) and \
not self._is_denied(target_endpoint, target_method, roles)
def _is_allowed(self, target_endpoint, target_method, user_roles):
return self._evaluate_permission_by_type(target_endpoint,
target_method, user_roles,
'allow')
def _is_denied(self, target_endpoint, target_method, user_roles):
return self._evaluate_permission_by_type(target_endpoint,
target_method, user_roles,
'deny')
def _evaluate_permission_by_type(self, target_endpoint, target_method,
user_roles, permission_type):
for role in user_roles:
role_permissions = self.permissions_by_roles.get(role,
{'allow': {},
'deny': {}})
relevant_permissions = role_permissions.get(permission_type, {})
if _is_permission_matching(target_endpoint, target_method,
relevant_permissions):
return True
return False
def start_watchdog():
event_handler = RankingHandler()
observer = Observer()
log_handler = LoggingEventHandler()
log_observer = Observer()
try:
observer.schedule(event_handler, path='./watch')
observer.start()
log_observer.schedule(log_handler, path='./watch')
log_observer.start()
logging.info("Watching Directory")
while True:
time.sleep(1)
except KeyboardInterrupt:
logging.info("Kill message sent. Aborting")
observer.stop()
log_observer.stop()
except:
logging.info("Unexpected error: %s" % sys.exc_info()[0])
observer.stop()
log_observer.stop()
error_message(sys.exc_info()[0])
observer.join()
log_observer.join()
def send_watchdog(self):
"""
Method of ``Mail``.
Send mail when new file created.
Alter this method if other condition of sending mail needed.
"""
#r_pipe,w_pipe = Pipe()
queue = Queue.Queue()
event = Event()
#event_handler = GetNameEventHandler(w_pipe,event)
#send_mail_thread = SendMailThread(r_pipe,event,self)
event_handler = GetNameEventHandler(queue,event)
send_mail_thread = SendMailThread(queue,event,self)
send_mail_thread.start()
observer = Observer()
path = self.config['BOOK_PATH']
observer.schedule(event_handler, path, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
event.set()
observer.stop()
#NOTICE:kill threads by force, need to change
sys.exit()
class FileEventHandler(FileSystemEventHandler):
def __init__(self):
FileSystemEventHandler.__init__(self)
def on_created(self,event):
global quiz
if event.src_path.split('/')[-1] == 'findQuiz':
with open('question.hortor.net/question/bat/findQuiz', encoding='utf-8') as f:
quiz=read_question(f)
elif event.src_path.split('/')[-1] == 'choose':
sql.sql_write(quiz)
elif event.src_path.split('/')[-1] == 'fightResult':
print('本局结束')
time.sleep(3)
adbshell.back()
time.sleep(3)
adbshell.tap('start')
if __name__ == "__main__":
observer = Observer()
event_handler = FileEventHandler()
observer.schedule(event_handler,'question.hortor.net/question/bat/',True)
print('-----答题器已运行，请开始排位-----')
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrup:
observer.stop()
observer.join()
elif (filename.endswith('.pdf')):
folder = 'Books'
elif (filename.endswith('.mp4')):
folder = 'Videos'
else:
folder = 'General'
create_folder_if_not_exists(folderToTrack + "/" + folder)
dist = folderToTrack + "/" + folder + "/" + filename
try:
shutil.move(src, dist)
except Exception:
print("An error happened")
print(Exception)
myHandler = FileSystemEventHandler()
myHandler.on_created = on_created
observer = Observer()
observer.schedule(myHandler, folderToTrack, recursive=True)
observer.start()
try:
print("Observing")
while True:
time.sleep(10)
except KeyboardInterrupt:
observer.stop()
observer.join()
if __name__ == "__main__":
FLAGS = gflags.FLAGS
gflags.DEFINE_string("state", ".",
"Path to state. Used to update metrics.")
gflags.DEFINE_string(
"root", "./runs", "Path to log directory. Will "
"track changes to any file in this directory.")
gflags.DEFINE_string("env", "main", "The env group for visdom.")
gflags.DEFINE_string("suffix", ".metric", "Suffix indicating metric file.")
gflags.DEFINE_boolean("verbose", False, "")
FLAGS(sys.argv)
if FLAGS.verbose:
logger.setLevel(logging.DEBUG)
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s [DEBUG] - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
event_handler = VisdomEventHandler(FLAGS.root, FLAGS.suffix, FLAGS.env)
observer = Observer()
observer.schedule(event_handler, FLAGS.root, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class Handler(FileSystemEventHandler):
def on_any_event(self, event):
for filename in os.listdir(settings.get('source')):
# Проверяем расширенеи файла
extension = filename.split(".")[-1]
# Если этот тип известен то
if len(extension) > 1 and (extension.lower() in target):
file = str(settings.get('source')) + filename
new_path = settings.get(target.get(extension.lower())) + filename
try:
os.rename(file, new_path)
except Exception:
pass
# Запуск всего на отслеживание
handle = Handler()
observer = Observer()
observer.schedule(handle, settings.get('source'), recursive=True)
observer.start()
# Программа будет срабатывать каждую секунду
try:
while True:
time.sleep(1000)
except KeyboardInterrupt:
observer.stop()
observer.join()
if len(sys.argv) == 2 and sys.argv[1] == 'kinect_calibrate':
calibrate_kinect_mode = True
if fake_mode == True:
from kinect import fakenect as fa
nect = fa.Fakenect()
else:
from kinect import kinect as ki
nect = ki.Kinect()
window = tkinter.Tk()
window.attributes('-fullscreen', True)
window.bind('<Escape>', lambda e: window.destroy())
sandbox = sb.Sandbox(config, nect, rd.Renderer(config))
if calibrate_beamer_mode == True:
sandbox.calibrate_beamer(window)
elif calibrate_kinect_mode == True:
sandbox.calibrate_kinect(window)
else:
sandbox.execute(window)
if __name__ == '__main__':
event_handler = ConfigChangeHandler()
observer = Observer()
observer.schedule(event_handler, path='config/', recursive=False)
observer.start()
main()
self.__close()
self.__open()
self.__read()
def on_created(self, event):
if event.src_path == self.filename and not event.is_directory:
logging.info("{0} created, open it".format(self.filename))
self.__close()
self.__open()
self.__read()
def on_modified(self, event):
if event.src_path == self.filename:
logging.info("{0} modified".format(event.src_path))
self.__read()
if __name__ == '__main__':
import sys
observer = Observer()
watcher = Watcher(sys.argv[1], None)
w2 = Watcher(sys.argv[2], None)
observer.schedule(watcher, watcher.dirname, recursive=False)
observer.schedule(w2, w2.dirname, recursive=False)
observer.start()
try:
observer.join()
except KeyboardInterrupt:
observer.stop()
observer.join()
with open("processing/test.csv", "w"):
pass
with open("processing/test1.csv", "w"):
pass
global changed
changed = False
#########################################################################################################
if __name__ == "__main__":
event_handler = MyHandler()
observer = Observer()
observer.schedule(event_handler, path='task_folder', recursive=False)
observer.start()
models.append(('LR', LogisticRegression()))
models.append(('LDA', LinearDiscriminantAnalysis()))
models.append(('KNN', KNeighborsClassifier()))
models.append(('CART', DecisionTreeClassifier()))
models.append(('NB', GaussianNB()))
models.append(('SVM', SVC()))
names = [
'metal1', 'metal2', 'metal3', 'metal4', 'metal5', 't1', 'c1', 't2',
'c2', 'rate'
]
names2 = ['metal1', 'metal2', 'metal3', 'metal4', 'metal5', 'rate']
while True:
message = "file "
message += f"{event.src_path} moved to {event.dest_path}"
print(message)
sendTasklist.append(createDatapack("move", event.src_path, event.dest_path))
event_handler = FileSystemEventHandler()
event_handler.on_created = on_created
event_handler.on_deleted = on_deleted
event_handler.on_modified = on_modified
event_handler.on_moved = on_moved
my_observer = Observer()
print("mpslsh", mountPointSlash)
while not os.path.exists(mountPointSlash):
time.sleep(0.1)
time.sleep(1) # must wait for juicefs to finish mounting
my_observer.schedule(event_handler, mountPointSlash, recursive=True)
my_observer.start()
print("Watchdog observer established")
async def login():
wsClient = await websockets.connect(wsUrl)
await wsClient.send(wsAuth)
return wsClient
async def wsSender(wsClient):
while True:
while len(sendTasklist) == 0:
await asyncio.sleep(0.1)
await wsClient.send(sendTasklist.popleft())
async def wsReceiver(wsClient):
"py": "Python",
}
for ext in ("dwg", "dxf"):
known_types[ext] = "AutoCAD"
ext_mapper = ExtensionMapper(known_types, "other", api_url, name_regex)
handler = NewFileHander(
ext_mapper,
args.destination,
excluded,
delay=args.delay,
ln_duration=args.ln_duration,
)
observer = Observer()
observer.schedule(handler, path=args.source, recursive=args.recursive)
observer.start()
if args.sort_old:
if args.recursive:
files_to_sort = [
os.path.join(dp, f)
for dp, dn, fn in os.walk(os.path.expanduser(args.source))
for f in fn
]
else:
files_to_sort = [
os.path.join(args.source, f)
for f in os.listdir(os.path.expanduser(args.source))
]
files_to_sort = [f for f in files_to_sort if os.path.isfile(f)]
# coding: utf-8
import sys
import time
from watchdog.events import PatternMatchingEventHandler
from watchdog.observers import Observer
import logging
logging.basicConfig(level=logging.DEBUG)
class MyEventHandler(PatternMatchingEventHandler):
def on_any_event(self, event):
logging.debug(event)
event_handler = MyEventHandler(patterns=['*.py', '*.pyc'],
ignore_patterns=['version.py'],
ignore_directories=True)
observer = Observer()
observer.schedule(event_handler, sys.argv[1], recursive=True)
observer.start()
try:
while True:
time.sleep(1)
finally:
observer.stop()
observer.join()
print("file created:{0}".format(event.src_path))
def on_deleted(self, event):
if event.is_directory:
print("directory deleted:{0}".format(event.src_path))
else:
print("file deleted:{0}".format(event.src_path))
def on_modified(self, event):
print(
"-----------------------Step on_modified--------------------------"
)
if event.is_directory:
print("directory modified:{0}".format(event.src_path))
else:
print("file modified:{0}".format(event.src_path))
if __name__ == "__main__":
dpath = "G:\\黄大宝python\\测试目录"
observer = Observer()
event_handler = FileEventHandler()
observer.schedule(event_handler, dpath, True)
observer.start()
print('------------------------------------------')
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
print("Model loaded.")
class ReloadModelHandler(FileSystemEventHandler):
def on_any_event(self, event):
# Reload model
print("Model update detected. Loading new model.")
time.sleep(MODEL_UPDATE_LAG_IN_SECONDS)
restoreVars()
loadModel()
# Setup watchdog
observer = Observer()
observer.schedule(ReloadModelHandler(), path=MODEL_DIR, recursive=False)
observer.start()
def classify(text):
text_series = pd.Series([text])
predict_x = np.array(list(vocab_processor.transform(text_series)))
print(predict_x)
y_predicted = [
p['class'] for p in classifier.predict(predict_x, as_iterable=True)
]
print(y_predicted[0])
topic = news_classes.class_map[str(y_predicted[0])]
return topic
print("directory created:{0}".format(event.src_path))
else:
print("file created:{0}".format(event.src_path))
def on_deleted(self, event):
if event.is_directory:
print("directory deleted:{0}".format(event.src_path))
else:
print("file deleted:{0}".format(event.src_path))
def on_modified(self, event):
if event.is_directory:
print("directory modified:{0}".format(event.src_path))
else:
print("file modified:{0}".format(event.src_path))
if __name__ == "__main__":
observer = Observer()
event_handler = FileEventHandler()
# 綁定監聽位置與監聽程序
observer.schedule(event_handler, '../Data/', True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
# 更新監聽
observer.join()
def run(self):
event_handler = FileChange()
observer = Observer()
observer.schedule(event_handler, path=self.path, recursive=True)
observer.start()
def start_observer(event_handler: LogFileHandler, logs_path: str) -> Observer:
observer = Observer()
observer.schedule(event_handler, logs_path, recursive=False)
observer.start()
return observer
l1_proc = Level1(CONFIG.get('Paths', 'tm_archive'),
CONFIG.get('Paths', 'fits_archive'))
l1_files = l1_proc.process_fits_files(files=l0_files)
logger.debug(l1_files)
if __name__ == '__main__':
tstart = time.perf_counter()
observer = Observer()
path = Path('/home/shane/tm')
soop_path = Path(CONFIG.get('Paths', 'soop_files'))
logging_handler = LoggingEventHandler(logger=logger)
tm_handler = GFTSFileHandler(process_tm, TM_REGEX)
# TODO should be an deticated path from the config
soop_manager = SOOPManager(soop_path)
soop_handler = GFTSFileHandler(soop_manager.add_soop_file_to_index,
SOOPManager.SOOP_FILE_REGEX)
observer.schedule(soop_handler, soop_manager.data_root, recursive=False)
observer.schedule(logging_handler, path, recursive=True)
observer.schedule(tm_handler, path, recursive=True)
observer.start()
try:
while True:
time.sleep(100)
except KeyboardInterrupt:
observer.stop()
observer.join()
class SqliteEventLogStorage(SqlEventLogStorage, ConfigurableClass):
"""SQLite-backed event log storage.
Users should not directly instantiate this class; it is instantiated by internal machinery when
``dagit`` and ``dagster-graphql`` load, based on the values in the ``dagster.yaml`` file in
``$DAGSTER_HOME``. Configuration of this class should be done by setting values in that file.
This is the default event log storage when none is specified in the ``dagster.yaml``.
To explicitly specify SQLite for event log storage, you can add a block such as the following
to your ``dagster.yaml``:
.. code-block:: YAML
run_storage:
module: dagster.core.storage.event_log
class: SqliteEventLogStorage
config:
base_dir: /path/to/dir
The ``base_dir`` param tells the event log storage where on disk to store the databases. To
improve concurrent performance, event logs are stored in a separate SQLite database for each
run.
"""
def __init__(self, base_dir, inst_data=None):
"""Note that idempotent initialization of the SQLite database is done on a per-run_id
basis in the body of connect, since each run is stored in a separate database."""
self._base_dir = os.path.abspath(check.str_param(base_dir, "base_dir"))
mkdir_p(self._base_dir)
self._watchers = defaultdict(dict)
self._obs = Observer()
self._obs.start()
self._inst_data = check.opt_inst_param(inst_data, "inst_data",
ConfigurableClassData)
def upgrade(self):
all_run_ids = self.get_all_run_ids()
print( # pylint: disable=print-call
"Updating event log storage for {n_runs} runs on disk...".format(
n_runs=len(all_run_ids)))
alembic_config = get_alembic_config(__file__)
for run_id in tqdm(all_run_ids):
with self.connect(run_id) as conn:
run_alembic_upgrade(alembic_config, conn, run_id)
@property
def inst_data(self):
return self._inst_data
@classmethod
def config_type(cls):
return {"base_dir": str}
@staticmethod
def from_config_value(inst_data, config_value):
return SqliteEventLogStorage(inst_data=inst_data, **config_value)
def get_all_run_ids(self):
all_filenames = glob.glob(os.path.join(self._base_dir, "*.db"))
return [
os.path.splitext(os.path.basename(filename))[0]
for filename in all_filenames
]
def path_for_run_id(self, run_id):
return os.path.join(self._base_dir,
"{run_id}.db".format(run_id=run_id))
def conn_string_for_run_id(self, run_id):
check.str_param(run_id, "run_id")
return create_db_conn_string(self._base_dir, run_id)
def _initdb(self, engine):
alembic_config = get_alembic_config(__file__)
try:
SqlEventLogStorageMetadata.create_all(engine)
engine.execute("PRAGMA journal_mode=WAL;")
stamp_alembic_rev(alembic_config, engine)
except (db.exc.DatabaseError, sqlite3.DatabaseError,
sqlite3.OperationalError) as exc:
# This is SQLite-specific handling for concurrency issues that can arise when, e.g.,
# the root nodes of a pipeline execute simultaneously on Airflow with SQLite storage
# configured and contend with each other to init the db. When we hit the following
# errors, we know that another process is on the case and it's safe to continue:
err_msg = str(exc)
if not ("table event_logs already exists" in err_msg
or "database is locked" in err_msg
or "table alembic_version already exists" in err_msg
or "UNIQUE constraint failed: alembic_version.version_num"
in err_msg):
raise
else:
logging.info(
"SqliteEventLogStorage._initdb: Encountered apparent concurrent init, "
"swallowing {str_exc}".format(str_exc=err_msg))
@contextmanager
def connect(self, run_id=None):
check.str_param(run_id, "run_id")
conn_string = self.conn_string_for_run_id(run_id)
engine = create_engine(conn_string, poolclass=NullPool)
if not os.path.exists(self.path_for_run_id(run_id)):
self._initdb(engine)
conn = engine.connect()
try:
with handle_schema_errors(
conn,
get_alembic_config(__file__),
msg="SqliteEventLogStorage for run {run_id}".format(
run_id=run_id),
):
yield conn
finally:
conn.close()
engine.dispose()
def wipe(self):
for filename in (glob.glob(os.path.join(self._base_dir, "*.db")) +
glob.glob(os.path.join(self._base_dir, "*.db-wal")) +
glob.glob(os.path.join(self._base_dir, "*.db-shm"))):
os.unlink(filename)
def watch(self, run_id, start_cursor, callback):
watchdog = SqliteEventLogStorageWatchdog(self, run_id, callback,
start_cursor)
self._watchers[run_id][callback] = (
watchdog,
self._obs.schedule(watchdog, self._base_dir, True),
)
def end_watch(self, run_id, handler):
if handler in self._watchers[run_id]:
event_handler, watch = self._watchers[run_id][handler]
self._obs.remove_handler_for_watch(event_handler, watch)
del self._watchers[run_id][handler]
class ConsolidatedSqliteEventLogStorage(AssetAwareSqlEventLogStorage,
ConfigurableClass):
"""SQLite-backed consolidated event log storage intended for test cases only.
Users should not directly instantiate this class; it is instantiated by internal machinery when
``dagit`` and ``dagster-graphql`` load, based on the values in the ``dagster.yaml`` file in
``$DAGSTER_HOME``. Configuration of this class should be done by setting values in that file.
To explicitly specify the consolidated SQLite for event log storage, you can add a block such as
the following to your ``dagster.yaml``:
.. code-block:: YAML
run_storage:
module: dagster.core.storage.event_log
class: ConsolidatedSqliteEventLogStorage
config:
base_dir: /path/to/dir
The ``base_dir`` param tells the event log storage where on disk to store the database.
"""
def __init__(self, base_dir, inst_data=None):
self._base_dir = check.str_param(base_dir, "base_dir")
self._conn_string = create_db_conn_string(base_dir,
SQLITE_EVENT_LOG_FILENAME)
self._secondary_index_cache = {}
self._inst_data = check.opt_inst_param(inst_data, "inst_data",
ConfigurableClassData)
self._watchdog = None
self._watchers = defaultdict(dict)
self._obs = Observer()
self._obs.start()
if not os.path.exists(self.get_db_path()):
self._init_db()
@property
def inst_data(self):
return self._inst_data
@classmethod
def config_type(cls):
return {"base_dir": StringSource}
@staticmethod
def from_config_value(inst_data, config_value):
return ConsolidatedSqliteEventLogStorage(inst_data=inst_data,
**config_value)
def _init_db(self):
mkdir_p(self._base_dir)
engine = create_engine(self._conn_string, poolclass=NullPool)
SqlEventLogStorageMetadata.create_all(engine)
engine.execute("PRAGMA journal_mode=WAL;")
alembic_config = get_alembic_config(__file__)
connection = engine.connect()
db_revision, head_revision = check_alembic_revision(
alembic_config, connection)
if not (db_revision and head_revision):
stamp_alembic_rev(alembic_config, engine)
@contextmanager
def connect(self, run_id=None):
engine = create_engine(self._conn_string, poolclass=NullPool)
conn = engine.connect()
try:
with handle_schema_errors(
conn,
get_alembic_config(__file__),
msg="ConsolidatedSqliteEventLogStorage requires migration",
):
yield conn
finally:
conn.close()
def get_db_path(self):
return os.path.join(self._base_dir,
"{}.db".format(SQLITE_EVENT_LOG_FILENAME))
def upgrade(self):
alembic_config = get_alembic_config(__file__)
with self.connect() as conn:
run_alembic_upgrade(alembic_config, conn)
def has_secondary_index(self, name, run_id=None):
if name not in self._secondary_index_cache:
self._secondary_index_cache[name] = super(
ConsolidatedSqliteEventLogStorage,
self).has_secondary_index(name, run_id)
return self._secondary_index_cache[name]
def enable_secondary_index(self, name, run_id=None):
super(ConsolidatedSqliteEventLogStorage,
self).enable_secondary_index(name)
if name in self._secondary_index_cache:
del self._secondary_index_cache[name]
def watch(self, run_id, start_cursor, callback):
if not self._watchdog:
self._watchdog = ConsolidatedSqliteEventLogStorageWatchdog(self)
watch = self._obs.schedule(self._watchdog, self._base_dir, True)
cursor = start_cursor if start_cursor is not None else -1
self._watchers[run_id][callback] = (cursor, watch)
def on_modified(self):
keys = [(run_id, callback)
for run_id, callback_dict in self._watchers.items()
for callback, _ in callback_dict.items()]
for run_id, callback in keys:
cursor, watch = self._watchers[run_id][callback]
# fetch events
events = self.get_logs_for_run(run_id, cursor)
# update cursor
self._watchers[run_id][callback] = (cursor + len(events), watch)
for event in events:
status = callback(event)
if (status == PipelineRunStatus.SUCCESS
or status == PipelineRunStatus.FAILURE
or status == PipelineRunStatus.CANCELED):
self.end_watch(run_id, callback)
def end_watch(self, run_id, handler):
if run_id in self._watchers:
_cursor, watch = self._watchers[run_id][handler]
self._obs.remove_handler_for_watch(self._watchdog, watch)
del self._watchers[run_id][handler]
if getext(event.src_path) in ('.wav'):
print('%s has been modified.' % event.src_path)
cmd = trans_cmd + event.src_path
print(cmd.split())
res = subprocess.check_call(cmd.split())
if getext(event.src_path) in ('.txt'):
print("voice.txt modified")
cmd = tweet_cmd
print(cmd.split())
res = subprocess.check_call(cmd.split())
def on_deleted(self, event):
if event.is_directory:
return
if getext(event.src_path) in ('.wav'):
print('%s has been deleted.' % event.src_path)
if __name__ in '__main__':
while 1:
event_handler = ChangeHandler()
observer = Observer()
observer.schedule(event_handler, BASEDIR, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class RunManager(object):
"""Manages a run's process, wraps its I/O, and synchronizes its files.
"""
def __init__(self,
api,
run,
project=None,
tags=[],
cloud=True,
job_type="train",
output=True,
port=None):
self._api = api
self._run = run
self._cloud = cloud
self._port = port
self._project = project if project else api.settings("project")
self._tags = tags
self._watch_dir = self._run.dir
self._config = run.config
self.job_type = job_type
self.url = self._run.get_url(api)
# We lock this when the backend is down so Watchdog will keep track of all
# the file events that happen. Then, when the backend comes back up, we unlock
# it so all the outstanding events will get handled properly. Watchdog's queue
# only keeps at most one event per file.
# Counterintuitively, we use the "reader" locking to guard writes to the W&B
# backend, and the "writer" locking to indicate that the backend is down. That
# way, users of the W&B API won't block each other, but can all be
# blocked by grabbing a "writer" lock.
self._file_event_lock = wandb.rwlock.RWLock()
# It starts acquired. We release it when we want to allow the events to happen.
# (ie. after the Run is successfully created)
self._file_event_lock.writer_enters()
self._event_handlers = {}
self._handler = PatternMatchingEventHandler()
self._handler.on_created = self._on_file_created
self._handler.on_modified = self._on_file_modified
self._handler._patterns = [
os.path.join(self._watch_dir, os.path.normpath('*'))
]
# Ignore hidden files/folders and output.log because we stream it specially
self._handler._ignore_patterns = [
'*/.*', '*.tmp',
os.path.join(self._run.dir, OUTPUT_FNAME)
]
self._observer = Observer()
self._observer.schedule(self._handler, self._watch_dir, recursive=True)
self._stats = stats.Stats()
# Calling .start() on _meta and _system_stats will spin a thread that reports system stats every 30 seconds
self._system_stats = stats.SystemStats(run, api)
self._meta = meta.Meta(api, self._run.dir)
self._meta.data["jobType"] = job_type
if self._run.program:
self._meta.data["program"] = self._run.program
self._file_pusher = file_pusher.FilePusher(self._push_function)
self._socket = wandb_socket.Client(self._port)
logger.debug("Initialized sync for %s/%s", self._project, self._run.id)
if self._cloud:
self._observer.start()
self._api.save_patches(self._watch_dir)
if output:
wandb.termlog("Syncing %s" % self.url)
wandb.termlog("Run `wandb off` to turn off syncing.")
wandb.termlog("Local directory: %s" % os.path.relpath(run.dir))
self._api.get_file_stream_api().set_file_policy(
OUTPUT_FNAME, CRDedupeFilePolicy())
""" FILE SYNCING / UPLOADING STUFF """
# TODO: limit / throttle the number of adds / pushes
def _on_file_created(self, event):
logger.info('file/dir created: %s', event.src_path)
if os.path.isdir(event.src_path):
return None
save_name = os.path.relpath(event.src_path, self._watch_dir)
self._file_event_lock.await_readable()
self._get_handler(event.src_path, save_name).on_created()
def _on_file_modified(self, event):
logger.info('file/dir modified: %s', event.src_path)
if os.path.isdir(event.src_path):
return None
save_name = os.path.relpath(event.src_path, self._watch_dir)
self._file_event_lock.await_readable()
self._get_handler(event.src_path, save_name).on_modified()
def _get_handler(self, file_path, save_name):
if not os.path.split(save_name)[0] == "media" and save_name not in [
'wandb-history.jsonl', 'wandb-events.jsonl',
'wandb-summary.json'
]:
# Don't show stats on media files
self._stats.update_file(file_path)
if save_name not in self._event_handlers:
if save_name == 'wandb-history.jsonl':
self._event_handlers[
'wandb-history.jsonl'] = FileEventHandlerTextStream(
file_path, 'wandb-history.jsonl', self._api)
elif save_name == 'wandb-events.jsonl':
self._event_handlers[
'wandb-events.jsonl'] = FileEventHandlerTextStream(
file_path, 'wandb-events.jsonl', self._api)
# Don't try to stream tensorboard files for now.
# elif 'tfevents' in save_name:
# # TODO: This is hard-coded, but we want to give users control
# # over streaming files (or detect them).
# self._api.get_file_stream_api().set_file_policy(save_name,
# BinaryFilePolicy())
# self._event_handlers[save_name] = FileEventHandlerBinaryStream(
# file_path, save_name, self._api)
# Overwrite handler (non-deferred) has a bug, wherein if the file is truncated
# during upload, the request to Google hangs (at least, this is my working
# theory). So for now we defer uploading everything til the end of the run.
# TODO: send wandb-summary during run. One option is to copy to a temporary
# file before uploading.
elif save_name == config.FNAME:
self._event_handlers[save_name] = FileEventHandlerConfig(
file_path, save_name, self._api, self._file_pusher,
self._run)
elif save_name == 'wandb-summary.json':
# Load the summary into the syncer process for meta etc to work
self._run.summary.load()
self._api.get_file_stream_api().set_file_policy(
save_name, OverwriteFilePolicy())
self._event_handlers[save_name] = FileEventHandlerSummary(
file_path, save_name, self._api, self._file_pusher,
self._run)
elif save_name.startswith('media/'):
# Save media files immediately
self._event_handlers[save_name] = FileEventHandlerOverwrite(
file_path, save_name, self._api, self._file_pusher)
else:
self._event_handlers[
save_name] = FileEventHandlerOverwriteDeferred(
file_path, save_name, self._api, self._file_pusher)
return self._event_handlers[save_name]
def _finish_handlers(self):
# TODO: there was a case where _event_handlers was getting modified in the loop.
for handler in list(self._event_handlers.values()):
handler.finish()
def _push_function(self, save_name, path):
with open(path, 'rb') as f:
self._api.push(self._project, {save_name: f},
run=self._run.id,
progress=lambda _, total: self._stats.
update_progress(path, total))
""" RUN MANAGEMENT STUFF """
def mirror_stdout_stderr(self):
"""Simple STDOUT and STDERR mirroring used by _init_jupyter"""
# TODO: Ideally we could start collecting logs without pushing
fs_api = self._api.get_file_stream_api()
io_wrap.SimpleTee(
sys.stdout,
streaming_log.TextStreamPusher(fs_api,
OUTPUT_FNAME,
prepend_timestamp=True))
io_wrap.SimpleTee(
sys.stderr,
streaming_log.TextStreamPusher(fs_api,
OUTPUT_FNAME,
prepend_timestamp=True,
line_prepend='ERROR'))
def _get_stdout_stderr_streams(self):
"""Sets up STDOUT and STDERR streams. Only call this once."""
if six.PY2 or "buffer" not in dir(sys.stdout):
stdout = sys.stdout
stderr = sys.stderr
else: # we write binary so grab the raw I/O objects in python 3
try:
stdout = sys.stdout.buffer.raw
stderr = sys.stderr.buffer.raw
except AttributeError:
# The testing environment and potentially others may have screwed with their
# io so we fallback to raw stdout / err
stdout = sys.stdout.buffer
stderr = sys.stderr.buffer
output_log_path = os.path.join(self._run.dir, OUTPUT_FNAME)
self._output_log = WriteSerializingFile(open(output_log_path, 'wb'))
stdout_streams = [stdout, self._output_log]
stderr_streams = [stderr, self._output_log]
if self._cloud:
# Tee stdout/stderr into our TextOutputStream, which will push lines to the cloud.
fs_api = self._api.get_file_stream_api()
self._stdout_stream = streaming_log.TextStreamPusher(
fs_api,
OUTPUT_FNAME,
prepend_timestamp=True,
lock_function=self._file_event_lock.reader_enters)
self._stderr_stream = streaming_log.TextStreamPusher(
fs_api,
OUTPUT_FNAME,
line_prepend='ERROR',
prepend_timestamp=True,
lock_function=self._file_event_lock.reader_enters)
stdout_streams.append(self._stdout_stream)
stderr_streams.append(self._stderr_stream)
return stdout_streams, stderr_streams
def _close_stdout_stderr_streams(self, exitcode):
self._output_log.f.close()
self._output_log = None
# Close output-capturing stuff. This also flushes anything left in the buffers.
if self._stdout_tee.tee_file is not None:
# we don't have tee_file's in headless mode
self._stdout_tee.tee_file.close()
# TODO(adrian): we should close these even in headless mode
# but in python 2 the read thread doesn't stop on its own
# for some reason
self._stdout_tee.close_join()
if self._stderr_tee.tee_file is not None:
self._stderr_tee.tee_file.close()
self._stderr_tee.close_join()
if self._cloud:
# not set in dry run mode
self._stdout_stream.close()
self._stderr_stream.close()
self._api.get_file_stream_api().finish(exitcode)
# Ensures we get a new file stream thread
self._api._file_stream_api = None
def _setup_resume(self, resume_status):
# write the tail of the history file
try:
history_tail = json.loads(resume_status['historyTail'])
jsonlfile.write_jsonl_file(
os.path.join(self._run.dir, wandb_run.HISTORY_FNAME),
history_tail)
except ValueError:
print("warning: couldn't load recent history")
# write the tail of the events file
try:
events_tail = json.loads(resume_status['eventsTail'])
jsonlfile.write_jsonl_file(
os.path.join(self._run.dir, wandb_run.EVENTS_FNAME),
events_tail)
except ValueError:
print("warning: couldn't load recent events")
# Note: these calls need to happen after writing the files above. Because the access
# to self._run.events below triggers events to initialize, but we need the previous
# events to be written before that happens.
# output.log
self._api.get_file_stream_api().set_file_policy(
OUTPUT_FNAME, CRDedupeFilePolicy(resume_status['logLineCount']))
# history
self._api.get_file_stream_api().set_file_policy(
wandb_run.HISTORY_FNAME,
DefaultFilePolicy(
start_chunk_id=resume_status['historyLineCount']))
self._event_handlers[
wandb_run.HISTORY_FNAME] = FileEventHandlerTextStream(
self._run.history.fname,
wandb_run.HISTORY_FNAME,
self._api,
seek_end=True)
# events
self._api.get_file_stream_api().set_file_policy(
wandb_run.EVENTS_FNAME,
DefaultFilePolicy(start_chunk_id=resume_status['eventsLineCount']))
self._event_handlers[
wandb_run.EVENTS_FNAME] = FileEventHandlerTextStream(
self._run.events.fname,
wandb_run.EVENTS_FNAME,
self._api,
seek_end=True)
def init_run(self, env=None):
self._system_stats.start()
self._meta.start()
self._api.get_file_stream_api().start()
if self._cloud:
storage_id = None
if self._run.resume != 'never':
resume_status = self._api.run_resume_status(
project=self._api.settings("project"),
entity=self._api.settings("entity"),
name=self._run.id)
if resume_status == None and self._run.resume == 'must':
raise LaunchError(
"resume='must' but run (%s) doesn't exist" %
self._run.id)
if resume_status:
print('Resuming run: %s' % self._run.get_url(self._api))
self._setup_resume(resume_status)
storage_id = resume_status['id']
if not self._upsert_run(False, storage_id, env):
self._upsert_run_thread = threading.Thread(
target=self._upsert_run, args=(True, storage_id, env))
self._upsert_run_thread.daemon = True
self._upsert_run_thread.start()
def shutdown(self, exitcode=0):
"""Stops system stats, streaming handlers, and uploads files without output, used by wandb.monitor"""
self._system_stats.shutdown()
self._meta.shutdown()
self._finish_handlers()
self._file_pusher.shutdown()
self._api.get_file_stream_api().finish(exitcode)
# Ensures we get a new file stream thread
self._api._file_stream_api = None
def _upsert_run(self, retry, storage_id, env):
"""Upsert the Run (ie. for the first time with all its attributes)
Arguments:
retry: (bool) Whether to retry if the connection fails (ie. if the backend is down).
False is useful so we can start running the user process even when the W&B backend
is down, and let syncing finish later.
Returns:
True if the upsert succeeded, False if it failed because the backend is down.
Throws:
LaunchError on other failures
"""
if retry:
num_retries = None
else:
num_retries = 0 # no retries because we want to let the user process run even if the backend is down
try:
upsert_result = self._run.save(id=storage_id,
num_retries=num_retries,
job_type=self.job_type,
api=self._api)
except wandb.api.CommError as e:
# TODO: Get rid of str contains check
if self._run.resume == 'never' and 'exists' in str(e):
raise LaunchError("resume='never' but run (%s) exists" %
self._run.id)
else:
if isinstance(e.exc, (requests.exceptions.HTTPError,
requests.exceptions.Timeout,
requests.exceptions.ConnectionError)):
wandb.termerror(
'Failed to connect to W&B. Retrying in the background.'
)
return False
raise LaunchError(
'Launch exception: {}, see {} for details. To disable wandb set WANDB_MODE=dryrun'
.format(e, util.get_log_file_path()))
self._run.set_environment(environment=env)
# unblock file syncing and console streaming, which need the Run to have a .storage_id
self._file_event_lock.writer_leaves()
return True
def run_user_process(self, program, args, env):
"""Launch a user process, capture its output, and sync its files to the backend.
This returns after the process has ended and syncing is done.
Captures ctrl-c's, signals, etc.
"""
stdout_streams, stderr_streams = self._get_stdout_stderr_streams()
if sys.platform == "win32":
# PTYs don't work in windows so we use pipes.
self._stdout_tee = io_wrap.Tee.pipe(*stdout_streams)
self._stderr_tee = io_wrap.Tee.pipe(*stderr_streams)
# Seems like the following actually isn't necessary on Windows
# TODO(adrian): we may need to do the following if we use pipes instead of PTYs
# because Python on Unix doesn't like writing UTF-8 to files
# tell child python interpreters we accept utf-8
# env['PYTHONIOENCODING'] = 'UTF-8'
else:
self._stdout_tee = io_wrap.Tee.pty(*stdout_streams)
self._stderr_tee = io_wrap.Tee.pty(*stderr_streams)
self._stdout_stream.write_string(
" ".join(psutil.Process(os.getpid()).cmdline()) + "\n\n")
command = [program] + list(args)
runner = util.find_runner(program)
if runner:
command = runner + command
command = ' '.join(six.moves.shlex_quote(arg) for arg in command)
try:
self.proc = subprocess.Popen(
command,
env=env,
stdout=self._stdout_tee.tee_file,
stderr=self._stderr_tee.tee_file,
shell=True,
)
except (OSError, IOError):
raise Exception('Could not find program: %s' % command)
self._sync_etc()
def wrap_existing_process(self,
pid,
stdout_read_fd,
stderr_read_fd,
port=None):
"""Do syncing, etc. for an already-running process.
This returns after the process has ended and syncing is done.
Captures ctrl-c's, signals, etc.
"""
stdout_read_file = os.fdopen(stdout_read_fd, 'rb')
stderr_read_file = os.fdopen(stderr_read_fd, 'rb')
stdout_streams, stderr_streams = self._get_stdout_stderr_streams()
self._stdout_tee = io_wrap.Tee(stdout_read_file, *stdout_streams)
self._stderr_tee = io_wrap.Tee(stderr_read_file, *stderr_streams)
self.proc = Process(pid)
try:
self.init_run()
except LaunchError as e:
wandb.termerror(str(e))
self._socket.launch_error()
return
# Signal the main process that we're all hooked up
self._socket.ready()
self._sync_etc(headless=True)
def _sync_etc(self, headless=False):
# Ignore SIGQUIT (ctrl-\). The child process will # handle it, and we'll
# exit when the child process does.
#
# We disable these signals after running the process so the child doesn't
# inherit this behaviour.
try:
signal.signal(signal.SIGQUIT, signal.SIG_IGN)
except AttributeError: # SIGQUIT doesn't exist on windows
pass
if self._api.update_available:
wandb.termlog(
"An update is available! To upgrade, please run:\n $ pip install wandb --upgrade"
)
# Add a space before user output
wandb.termlog()
if env.get_show_run():
webbrowser.open_new_tab(self._run.get_url(self._api))
exitcode = None
try:
while True:
res = bytearray()
try:
res = self._socket.recv(2)
except socket.timeout:
pass
if len(res) == 2 and res[0] == 2:
exitcode = res[1]
break
elif len(res) > 0:
wandb.termerror(
"Invalid message received from child process: %s" %
str(res))
break
else:
exitcode = self.proc.poll()
if exitcode is not None:
break
time.sleep(1)
except KeyboardInterrupt:
exitcode = 255
if headless:
wandb.termlog('Ctrl-c pressed.')
else:
wandb.termlog(
'Ctrl-c pressed; waiting for program to end. Press ctrl-c again to kill it.'
)
try:
while self.proc.poll() is None:
time.sleep(0.1)
except KeyboardInterrupt:
pass
if self.proc.poll() is None:
wandb.termlog('Program still alive. Killing it.')
try:
self.proc.kill()
except OSError:
pass
"""TODO(adrian): garbage that appears in the logs sometimes
Exception ignored in: <bound method Popen.__del__ of <subprocess.Popen object at 0x111adce48>>
Traceback (most recent call last):
File "/Users/adrian/.pyenv/versions/3.6.0/Python.framework/Versions/3.6/lib/python3.6/subprocess.py", line 760, in __del__
AttributeError: 'NoneType' object has no attribute 'warn'
"""
if exitcode is None:
exitcode = 254
wandb.termlog(
'Killing program failed; syncing files anyway. Press ctrl-c to abort syncing.'
)
else:
if exitcode == 0:
wandb.termlog('Program ended.')
else:
wandb.termlog(
'Program failed with code %d. Press ctrl-c to abort syncing.'
% exitcode)
#termlog('job (%s) Process exited with code: %s' % (program, exitcode))
self._meta.data["exitcode"] = exitcode
if exitcode == 0:
self._meta.data["state"] = "finished"
elif exitcode == 255:
self._meta.data["state"] = "killed"
else:
self._meta.data["state"] = "failed"
self._meta.shutdown()
self._system_stats.shutdown()
if exitcode != 0 and START_TIME - time.time() < 30:
wandb.termlog("Process crashed early, not syncing files")
sys.exit(exitcode)
# TODO: these can be slow to complete
self._close_stdout_stderr_streams(exitcode)
# If we're not syncing to the cloud, we're done
if not self._cloud:
sys.exit(exitcode)
# Show run summary/history
self._run.summary.load()
summary = self._run.summary._summary
if len(summary):
wandb.termlog('Run summary:')
max_len = max([len(k) for k in summary.keys()])
format_str = ' {:>%s} {}' % max_len
for k, v in summary.items():
# arrays etc. might be too large. for now we just don't print them
if isinstance(v, six.string_types):
if len(v) >= 20:
v = v[:20] + '...'
wandb.termlog(format_str.format(k, v))
elif isinstance(v, numbers.Number):
wandb.termlog(format_str.format(k, v))
self._run.history.load()
history_keys = self._run.history.keys()
if len(history_keys):
wandb.termlog('Run history:')
max_len = max([len(k) for k in history_keys])
for key in history_keys:
vals = util.downsample(self._run.history.column(key), 40)
if any((not isinstance(v, numbers.Number) for v in vals)):
continue
line = sparkline.sparkify(vals)
format_str = u' {:>%s} {}' % max_len
wandb.termlog(format_str.format(key, line))
if self._run.has_examples:
wandb.termlog('Saved %s examples' % self._run.examples.count())
wandb.termlog('Waiting for final file modifications.')
# This is a a heuristic delay to catch files that were written just before
# the end of the script.
# TODO: ensure we catch all saved files.
# TODO(adrian): do we need this?
time.sleep(2)
try:
# avoid hanging if we crashed before the observer was started
if self._observer.is_alive():
self._observer.stop()
self._observer.join()
# TODO: py2 TypeError: PyCObject_AsVoidPtr called with null pointer
except TypeError:
pass
# TODO: py3 SystemError: <built-in function stop> returned a result with an error set
except SystemError:
pass
self._finish_handlers()
self._file_pusher.finish()
wandb.termlog('Syncing files in %s:' %
os.path.relpath(self._watch_dir))
for file_path in self._stats.files():
wandb.termlog(' %s' % os.path.relpath(file_path, self._watch_dir))
step = 0
spinner_states = ['-', '\\', '|', '/']
stop = False
self._stats.update_all_files()
while True:
if not self._file_pusher.is_alive():
stop = True
summary = self._stats.summary()
line = (
' %(completed_files)s of %(total_files)s files,'
' %(uploaded_bytes).03f of %(total_bytes).03f bytes uploaded\r'
% summary)
line = spinner_states[step % 4] + line
step += 1
wandb.termlog(line, newline=False)
if stop:
break
time.sleep(0.25)
#print('FP: ', self._file_pusher._pending, self._file_pusher._jobs)
# clear progress line.
wandb.termlog(' ' * 79)
# Check md5s of uploaded files against what's on the file system.
# TODO: We're currently using the list of uploaded files as our source
# of truth, but really we should use the files on the filesystem
# (ie if we missed a file this wouldn't catch it).
# This polls the server, because there a delay between when the file
# is done uploading, and when the datastore gets updated with new
# metadata via pubsub.
wandb.termlog('Verifying uploaded files... ', newline=False)
error = False
mismatched = None
for delay_base in range(4):
mismatched = []
download_urls = self._api.download_urls(self._project,
run=self._run.id)
for fname, info in download_urls.items():
if fname == 'wandb-history.h5' or OUTPUT_FNAME:
continue
local_path = os.path.join(self._watch_dir, fname)
local_md5 = util.md5_file(local_path)
if local_md5 != info['md5']:
mismatched.append((local_path, local_md5, info['md5']))
if not mismatched:
break
wandb.termlog(' Retrying after %ss' % (delay_base**2))
time.sleep(delay_base**2)
if mismatched:
print('')
error = True
for local_path, local_md5, remote_md5 in mismatched:
wandb.termerror(
'%s (%s) did not match uploaded file (%s) md5' %
(local_path, local_md5, remote_md5))
else:
print('verified!')
if error:
wandb.termerror('Sync failed %s' % self.url)
else:
wandb.termlog('Synced %s' % self.url)
sys.exit(exitcode)
class GitWatcher(object):
def __init__(self,
path=None,
events_timeout=1,
notifier=None,
notify_callback=None):
self.event_handler = MyEventHandler(ignore_patterns=IGNORE_PATTERNS,
parent=self)
self.path = path
self.events_timeout = events_timeout
self.observer = Observer()
self.observer.schedule(self.event_handler, self.path, recursive=True)
self.observer.start()
self.events_list = []
self.stopped = False
self.git_handler = GitDirectoryHandler(path=self.path, parent=self)
self._thread = Thread(target=self._run)
self.notifier = notifier
self.notify_callback = notify_callback
if self.notifier:
self.notifier.parent = self
def on_notify(self, message):
print(message)
if self.notify_callback:
self.notify_callback('Синхронизация с сервером')
self.git_handler.update_from_remote()
if self.notify_callback:
self.notify_callback('Синхронизация с сервером выполнена')
def on_push(self):
self.notifier.send_notify('hello')
if self.notify_callback:
self.notify_callback('Отправка файлов')
def on_any_event(self, event):
self.events_list.append({
'event': event,
'timestamp': datetime.datetime.now()
})
def _run(self):
while not self.stopped:
time.sleep(0.1)
if len(self.events_list) > 0:
last_event = self.events_list[-1]
if (datetime.datetime.now() - last_event['timestamp']
).total_seconds() > self.events_timeout:
self.process_events()
@staticmethod
def touch(path):
with open(path, 'w'):
pass
@staticmethod
def get_empty_file_path(path):
return os.path.join(path, '.empty')
def handle_empty_directory(self, path):
dir_list = os.listdir(path)
if len(dir_list) == 0:
self.touch(self.get_empty_file_path(path))
elif os.path.isfile(
self.get_empty_file_path(path)) and len(dir_list) > 1:
os.unlink(self.get_empty_file_path(path))
def process_events(self):
modified_paths = {}
for event in self.events_list:
if event['event'].src_path in modified_paths:
if (EVENTS_PRIORITIES.get(event['event'].event_type, 100) >
EVENTS_PRIORITIES.get(
modified_paths[event['event'].src_path].event_type,
0)):
modified_paths[event['event'].src_path] = event['event']
else:
modified_paths[event['event'].src_path] = event['event']
modified_files = []
for k, v in modified_paths.items():
print(k, v)
if v.is_directory and os.path.isdir(k):
self.handle_empty_directory(k)
if os.path.basename(k) == '.empty' or (v.is_directory and
v.event_type == 'modified'):
continue
modified_files.append('[{} {}]'.format(
EVENTS_ABBR.get(v.event_type), os.path.relpath(k, self.path)))
self.git_handler.process_changes()
self.events_list = []
def start(self):
self._thread.start()
def stop(self):
self.stopped = True
self._thread.join()
self.observer.stop()
self.observer.join()
if self.notifier:
self.notifier.stop()
def __del__(self):
self.observer.stop()
self.observer.join()
app = Flask(__name__)
image_path = "./image_test"
config_path = "./config/config.ini"
config = Config.read_config(config_path)
@app.route("/")
def top():
return render_template("index.html")
@app.route("/config")
def config():
return render_template("config.html")
@app.route("/save_config", methods=["POST"])
def save_config():
return render_template("index.html")
if __name__ == "__main__":
print("on app")
change_handler = ChangeHandler()
observer = Observer()
observer.schedule(change_handler, image_path, recursive=True)
observer.start()
app.run(host="127.0.0.1", port=5000)
{'id': xmlid,
'name': '%s.xml' % toolname,
'src': 'hda', 'file_type':'xml', 'visible':'true'},
{'id': lintid,
'name': '%s Planemo lint report' % toolname,
'src': 'hda', 'file_type':'txt', 'visible':'true'}],
'name': '%s test reports and tested toolshed archive' % toolname}
resdict = gi.histories.create_dataset_collection(hid, newcoll)
logging.info('Create_results added collection with results')
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
# filename='/export/galaxy/toolwatcher.log'),
# filemode="w",
)
event_handler = ToolHandler(watchme=WATCHME, planemo_galaxy_root=PLANEMO_GALAXY_ROOT,
conda_prefix=CONDA_PREFIX, patterns=WATCH_PATTERN, galaxy_root=GALAXY_ROOT)
observer = Observer()
observer.schedule(event_handler, path=WATCHME, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class MyHandler(FileSystemEventHandler):
file_format_correct = True
# file has been created
def on_created(self, event):
if not os.path.isdir(event.src_path):
self.file_format_correct = handle_bill_move(event)
# if a file is modified
def on_modified(self, event):
if self.file_format_correct and not os.path.isdir(event.src_path):
self.file_format_correct = handle_bill_move(event)
event_handler = MyHandler()
# starting the observer and keep it running until you enter "control + c"
observer = Observer()
observer.schedule(event_handler, track_folder, recursive=True)
observer.start()
write_log("Watcher started!")
try:
while True:
time.sleep(10)
except KeyboardInterrupt:
observer.stop()
observer.join()
class Library(RegexMatchingEventHandler):
def __init__(self, **kwargs):
"""
Initializes a new instance of the :class:`Library` class.
"""
self._supported = kwargs.get('supported', ['.mp3', '.ogg'])
self._base_path = os.path.join(os.path.expanduser(kwargs.get('base_path', '~/Music')), '')
self._libraries = list()
self._current_library = 0
self._current_song = -1
self.on_changed = lambda *a, **kw: None
regexes = [
r'{root_dir}'.format(
root_dir=os.path.abspath(os.path.join(self._base_path, os.pardir)).replace('/', '\/')
)
]
super(Library, self).__init__(regexes=regexes)
def __enter__(self):
"""Starts the song library"""
self._rescan_library()
self._observer = Observer()
parent = os.path.abspath(os.path.join(self._base_path, os.pardir))
self._observer.schedule(self, path=parent, recursive=True)
self._observer.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Stops the song library"""
self._observer.stop()
self._observer.join()
@property
def library(self):
"""Gets the current selected song library"""
return self._current_library
@library.setter
def library(self, value):
"""Selects a song library"""
if not self._libraries:
return
self._current_library = value % len(self._libraries)
self._current_song = -1
@property
def song(self):
"""Gets the current selected song of the current song library"""
if self._current_library >= len(self._libraries):
return
if self._current_song >= len(self._libraries[self._current_library]):
return
return self._libraries[self._current_library][self._current_song]
def next(self):
"""Selects the next song in the current song library"""
if not self._libraries:
return
self._current_song = (self._current_song + 1) % len(self._libraries[self._current_library])
def _rescan_library(self):
"""Scans the root directory for song libraries"""
self._libraries.clear()
logger.info('loading audio library %s', self._base_path)
if not os.path.isdir(self._base_path):
logger.info('audio library is empty')
self.on_changed()
return
for index, directory in enumerate(sorted(os.listdir(self._base_path))):
directory = os.path.join(self._base_path, directory)
if not os.path.isdir(directory):
continue
logger.info('adding songs from directory %s', directory)
self._libraries.append(sorted(
[os.path.join(directory, file)
for file in os.listdir(directory)
if os.path.isfile(os.path.join(directory, file))
and os.path.splitext(file.lower())[1] in self._supported]
))
self.library = self._current_library
self.on_changed()
def on_any_event(self, event):
if isinstance(event, (DirCreatedEvent, DirModifiedEvent, DirDeletedEvent)):
self._rescan_library()
elif isinstance(event, FileCreatedEvent):
self._on_file_created(str(event.src_path))
elif isinstance(event, FileMovedEvent):
self._on_file_removed(str(event.src_path))
self._on_file_created(str(event.dest_path))
def _on_file_removed(self, file_path):
changed = False
for index, library in enumerate(self._libraries[:]):
if file_path in library:
logger.info('remove song from directory %s', file_path)
library.remove(file_path)
changed = True
if not library:
self._libraries.remove(library)
self.on_changed()
def _on_file_created(self, file_path):
dir_name, file_name = os.path.split(file_path)
if not dir_name.startswith(self._base_path):
return
library = [i for i, x in enumerate(self._libraries) if x[0].startswith(dir_name)]
if len(library) > 1:
logging.warning('can\'t add song, multiple libraries matches: %s', file_path)
return
if len(library) == 0:
logging.info('add new library: %s', dir_name)
self._libraries.append([file_path])
self._libraries = sorted(self._libraries)
self.on_changed()
return
index = library[0]
if file_path not in self._libraries[index]:
logging.info('add song to library: %s', file_path)
library = self._libraries[index]
library.append(file_path)
self._libraries[index] = sorted(library)
self.on_changed()
def __str__(self):
return '%s' % self._libraries
self.ch = slack_channel
def on_created(self, event):
# slack.chat.post_message('self.ch', event.src_path)
slack.files.upload(event.src_path, channels=self.ch)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--token', help='api_token from slack')
parser.add_argument('--ch', help='channel from slack')
parser.add_argument('--path', help='directory to monitor')
args = parser.parse_args()
print(
'Monitoring directory {} \n Slack token {} \n Slack channel {}'.format(
args.path, args.token, args.ch))
slack = Slacker(args.token)
event_handler = MyHandler(args.ch)
observer = Observer()
observer.schedule(event_handler, path=args.path, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
import sys
import time
import logging
from watchdog.observers import Observer
from watchdog.events import LoggingEventHandler
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
path = sys.argv[1] if len(sys.argv) > 1 else '.'
event_handler = LoggingEventHandler()
observer = Observer()
observer.schedule(event_handler, path, recursive=T)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def _execute(self, options, args):
"""Start the watcher."""
self.logger = get_logger('auto', STDERR_HANDLER)
LRSocket.logger = self.logger
if WebSocket is object and watchdog is None:
req_missing(['ws4py', 'watchdog'], 'use the "auto" command')
elif WebSocket is object:
req_missing(['ws4py'], 'use the "auto" command')
elif watchdog is None:
req_missing(['watchdog'], 'use the "auto" command')
self.cmd_arguments = ['nikola', 'build']
if self.site.configuration_filename != 'conf.py':
self.cmd_arguments.append('--conf=' +
self.site.configuration_filename)
# Run an initial build so we are up-to-date
subprocess.call(self.cmd_arguments)
port = options and options.get('port')
self.snippet = '''<script>document.write('<script src="http://'
+ (location.host || 'localhost').split(':')[0]
+ ':{0}/livereload.js?snipver=1"></'
+ 'script>')</script>
</head>'''.format(port)
# Do not duplicate entries -- otherwise, multiple rebuilds are triggered
watched = set(['templates/'] +
[get_theme_path(name) for name in self.site.THEMES])
for item in self.site.config['post_pages']:
watched.add(os.path.dirname(item[0]))
for item in self.site.config['FILES_FOLDERS']:
watched.add(item)
for item in self.site.config['GALLERY_FOLDERS']:
watched.add(item)
for item in self.site.config['LISTINGS_FOLDERS']:
watched.add(item)
for item in self.site._plugin_places:
watched.add(item)
# Nikola itself (useful for developers)
watched.add(pkg_resources.resource_filename('nikola', ''))
out_folder = self.site.config['OUTPUT_FOLDER']
if options and options.get('browser'):
browser = True
else:
browser = False
if options['ipv6']:
dhost = '::'
else:
dhost = None
host = options['address'].strip('[').strip(']') or dhost
# Server can be disabled (Issue #1883)
self.has_server = not options['no-server']
# Instantiate global observer
observer = Observer()
if self.has_server:
# Watch output folders and trigger reloads
observer.schedule(OurWatchHandler(self.do_refresh),
out_folder,
recursive=True)
# Watch input folders and trigger rebuilds
for p in watched:
if os.path.exists(p):
observer.schedule(OurWatchHandler(self.do_rebuild),
p,
recursive=True)
# Watch config file (a bit of a hack, but we need a directory)
_conf_fn = os.path.abspath(self.site.configuration_filename
or 'conf.py')
_conf_dn = os.path.dirname(_conf_fn)
observer.schedule(ConfigWatchHandler(_conf_fn, self.do_rebuild),
_conf_dn,
recursive=False)
try:
self.logger.info("Watching files for changes...")
observer.start()
except KeyboardInterrupt:
pass
parent = self
class Mixed(WebSocketWSGIApplication):
"""A class that supports WS and HTTP protocols on the same port."""
def __call__(self, environ, start_response):
if environ.get('HTTP_UPGRADE') is None:
return parent.serve_static(environ, start_response)
return super(Mixed, self).__call__(environ, start_response)
if self.has_server:
ws = make_server(host,
port,
server_class=WSGIServer,
handler_class=WebSocketWSGIRequestHandler,
app=Mixed(handler_cls=LRSocket))
ws.initialize_websockets_manager()
self.logger.info("Serving HTTP on {0} port {1}...".format(
host, port))
if browser:
if options['ipv6'] or '::' in host:
server_url = "http://[{0}]:{1}/".format(host, port)
else:
server_url = "http://{0}:{1}/".format(host, port)
self.logger.info(
"Opening {0} in the default web browser...".format(
server_url))
# Yes, this is racy
webbrowser.open('http://{0}:{1}'.format(host, port))
try:
self.dns_sd = dns_sd(port, (options['ipv6'] or '::' in host))
ws.serve_forever()
except KeyboardInterrupt:
self.logger.info("Server is shutting down.")
if self.dns_sd:
self.dns_sd.Reset()
# This is a hack, but something is locking up in a futex
# and exit() doesn't work.
os.kill(os.getpid(), 15)
else:
# Workaround: can’t have nothing running (instant exit)
# but also can’t join threads (no way to exit)
# The joys of threading.
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
self.logger.info("Shutting down.")
# This is a hack, but something is locking up in a futex
# and exit() doesn't work.
os.kill(os.getpid(), 15)
class MakeMKV_Watchdog(FileSystemEventHandler):
def __init__(self, *args, **kwargs):
super().__init__()
self.log = logging.getLogger(__name__)
self.log.info('Starting up...')
fileExt = kwargs.pop(
'fileExt',
None) # Try to get fileExt keyword from keyword dictionary
if (fileExt is None): # If fileExt is None
self.fileExt = ('.mkv',
) # Set to fileExt attribute to default of '.mkv'
elif isinstance(fileExt, str): # Else, if it is string
self.fileExt = (fileExt,
) # Set fileExt attribute to tuple of fileExt
elif isinstance(fileExt, list): # Else, if it is list
self.fileExt = tuple(
fileExt) # Set fileExt attribute to tuple of fileExt
else: # Else
self.fileExt = fileExt # Set fileExt attribute using fileExt keyword value
self.converter = VideoConverter(**kwargs)
self.Queue = Queue(
) # Initialize queue for sending files to converting thread
self.Observer = Observer() # Initialize a watchdog Observer
for arg in args: # Iterate over input arguments
self.Observer.schedule(
self, arg, recursive=True
) # Add each input argument to observer as directory to watch; recursively
for file in self._getDirListing(
arg
): # Iterate over all files (if any) in the input directory
self.Queue.put(file) # Enqueue the file
self.Observer.start() # Start the observer
self.__runThread = Thread(
target=self.__run) # Thread for dequeuing files and converting
self.__runThread.start() # Start the thread
def on_created(self, event):
'''
Purpose:
Method to handle events when file is created.
'''
if event.is_directory: return # If directory; just return
if event.src_path.endswith(self.fileExt):
self.Queue.put(
event.src_path
) # Add split file path (dirname, basename,) tuple to to_convert list
self.log.debug('New file added to queue : {}'.format(
event.src_path)) # Log info
def on_moved(self, event):
'''
Purpose:
Method to handle events when file is created.
'''
if event.is_directory: return # If directory; just return
if event.dest_path.endswith(self.fileExt):
self.Queue.put(
event.dest_path
) # Add split file path (dirname, basename,) tuple to to_convert list
self.log.debug('New file added to queue : {}'.format(
event.dest_path)) # Log info
def join(self):
'''
Method to wait for the watchdog Observer to finish.
The Observer will be stopped when _sigintEvent or _sigtermEvent is set
'''
self.Observer.join() # Join the observer thread
def _getDirListing(self, dir):
'''
Purpose:
Method to get list of files in a directory (non-recursive), that
ends with given extension
Inputs:
dir : Path of directory to search
Keywords:
ext : Tuple of extensions to check, default = ('.mkv',)
Outputs:
Returns list of files
'''
files = []
for file in os.listdir(dir):
if file.endswith(self.fileExt):
path = os.path.join(dir, file)
if os.path.isfile(path):
files.append(path)
return files
def _checkSize(self, file):
'''
Purpose:
Method to check that file size has stopped changing
Inputs:
file : Full path to a file
Outputs:
None.
Keywords:
None.
'''
self.log.debug('Waiting for file to finish being created')
prev = -1 # Set previous file size to -1
curr = os.path.getsize(file) # Get current file size
while (prev != curr): # While sizes differ
time.sleep(SLEEP) # Sleep a few seconds seconds
prev = curr # Set previous size to current size
curr = os.path.getsize(file) # Update current size
def __run(self, **kwargs):
'''
Purpose:
A thread to dequeue video file paths and convert them
Inputs:
None.
Outputs:
None.
Keywords:
None.
'''
while isRunning(): # While the kill event is NOT set
try: # Try
file = self.Queue.get(
timeout=TIMEOUT
) # Get a file from the queue; block for 0.5 seconds then raise exception
except: # Catch exception
continue # Do nothing
self._checkSize(
file) # Wait to make sure file finishes copying/moving
try:
out_file = self.converter.transcode(file) # Convert file
except:
self.log.exception('Failed to convert file')
else:
if out_file is not None and isRunning():
plexMediaScanner(
'scan',
'refresh',
section='TV Shows'
if self.converter.metaData.isEpisode else 'Movies')
self.Queue.task_done()
self.log.info('MakeMKV watchdog stopped!')
self.Observer.stop()
#System
'.bak': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.cab': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.cfg': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.cpl': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.cur': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.dll': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.dmp': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.drv': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.icns': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.ico': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.ini': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.lnk': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.msi': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.sys': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
'.tmp': "/Users/emreakayoglu/Downloads/TEST/Text/Other/System",
}
folder_to_track = '/Users/emreakayoglu/Desktop/TEST'
folder_destination = '/Users/emreakayoglu/Downloads/TEST'
event_handler = MyHandler()
observer = Observer()
observer.schedule(event_handler, folder_to_track, recursive=True)
observer.start()
try:
while True:
time.sleep(10)
except KeyboardInterrupt:
observer.stop()
observer.join()
def mainLoop(self):
logging.debug("Monitor: started main loop.")
self.session = self.dm.Session()
self.library = Library(self.dm.Session)
observer = Observer()
self.eventHandler = MonitorEventHandler(self)
for path in self.paths:
if os.path.exists(path):
observer.schedule(self.eventHandler, path, recursive=True)
observer.start()
while True:
try:
(msg, args) = self.queue.get(block=True, timeout=1)
except:
msg = None
#dispatch messages
if msg == "scan":
self.dofullScan(self.paths)
if msg == "events":
self.doEventProcessing(args)
#time.sleep(1)
if self.quit:
break
self.session.close()
self.session = None
observer.stop()
logging.debug("Monitor: stopped main loop.")
