def ds_addlog(source, type, user, character):
user = User.objects.get(id=user)
log = Log(source=source,
type=type,
user=user,
character=character)
log.save()
return "OK!" + "from " + log.source + "log.user" + " logged in"
class process():
"""
Abstract class for processing.
Provides basic options and itarators.
Only operates on episodes in ready_state,
or all if ready_state is None.
if read_sate is not None and .processing() returns True,
bump the episode's state:
state=ready_state+1
"""
extra_options = {}
# set stop to True to stop processing at the end of episode
# will not abort a process, like an encode, upload or anything like that.
# it just exits these loops:
# for e in es: (process episodes) and do while not Done: (poll)
stop = False
ready_state = None
# defaults to ntsc stuff
# fps=29.98
fps = 30000 / 1001.0
bpf = 120000
def run_cmd(self, cmd):
"""
given command list
if verbose prints stuff,
runs it, returns pass/fail.
"""
if self.options.verbose:
print(cmd)
print(' '.join(cmd))
if self.options.test:
print("TEST: not running command.")
return True
p = subprocess.Popen(cmd)
p.wait()
retcode = p.returncode
if retcode:
if self.options.verbose:
print("command failed")
print(retcode)
ret = False
else:
ret = True
return ret
def run_cmds(self, episode, cmds):
"""
given a list of commands
append them to the episode's shell script
then run each
abort and return False if any fail.
"""
script_pathname = os.path.join(self.work_dir, episode.slug + ".sh")
sh = open(script_pathname, 'a')
for cmd in cmds:
if type(cmd) == list:
print(cmd)
log_text = ' '.join(cmd)
else:
log_text = cmd
cmd = cmd.split()
try:
# Not sure what python v3 wants. str I guess.
# .write(b'a') and .writeline(b'a') - Errpr must be str, not b
sh.writelines(log_text)
# sh.writelines([t.encode('utf-8','ignore') for t in log_text])
except Exception as e:
import code
code.interact(local=locals())
import sys
sys.exit()
# if isinstance(s, unicode): return s.encode('utf-8') else: return s
sh.write('\n')
# self.log_info(log_text)
if self.options.debug_log:
episode.description += "\n{0:>s}\n".format(log_text, )
episode.save()
if not self.run_cmd(cmd):
return False
sh.write('\n')
sh.close()
return True
def file2cdn(self, show, src, dst=None):
"""
src is relitive to the show dir.
src and dst get filled to full paths.
Check to see if src exists,
if it does, try to upload it to cdn
(rax_uploader will skip if same file exists).
"""
print("checking:", src)
if dst is None: dst = src
src = os.path.join(self.show_dir, src)
dst = os.path.join("veyepar", show.client.slug, show.slug, dst)
if os.path.exists(src):
u = rax_uploader.Uploader()
u.user = show.client.rax_id
if not show.client.bucket_id.strip():
raise AttributeError("client.bucket_id is blank")
u.bucket_id = show.client.bucket_id
u.bucket = show.client.bucket_id
u.pathname = src
u.key_id = dst
ret = u.upload()
print(u.new_url)
ret = u.new_url
else:
print(("file2cdn can't find {}".format(src)))
ret = False
return ret
def set_dirs(self, show):
client = show.client
self.show_dir = os.path.join(self.options.media_dir, client.slug,
show.slug)
if self.options.tmp_dir:
self.tmp_dir = self.options.tmp_dir
else:
self.tmp_dir = os.path.join(self.show_dir, "tmp")
# change me to "cmd" or something before the next show
self.work_dir = os.path.join(self.show_dir, "tmp")
return
def log_in(self, episode):
"""
log_in/out
create logs, and lock.unlock the episode
"""
where = socket.gethostname()
if os.environ.get('STY') is not None:
where += ":" + os.environ['STY']
what_where = "%s@%s" % (self.__class__.__name__, where)
episode.locked = datetime.datetime.now()
episode.locked_by = what_where
# save the lock to disk
episode.save()
try:
state = State.objects.get(sequence=episode.state)
except State.DoesNotExist:
state = None
self.start = datetime.datetime.now()
self.log = Log(episode=episode,
state=state,
ready=datetime.datetime.now(),
start=datetime.datetime.now(),
result=what_where)
self.log.save()
def log_info(self, text):
self.log.result += text + '\n'
def log_out(self, episode):
# done processing
# log the end time
# unlock the episode
self.log.end = datetime.datetime.now()
self.log.save()
episode.locked = None
episode.locked_by = ''
episode.save()
def process_ep(self, episode):
print(("stubby process_ep: #{} state:{} {}".format(
episode.id, episode.state, episode.name)))
return
def process_eps(self, episodes):
def foo(e):
# shard over ...umm.. a bunch of tmux sessions I think.
s = os.environ.get('STY')
if s:
s = s.split('-')
if len(s) == 3:
s = s[-1]
if s.isdigit():
return e.id % 11 == int(s)
return True
ret = None
sleepytime = False
for e in episodes:
if foo(e):
# next line requires the db to make sure the lock field is fresh
ep = Episode.objects.get(pk=e.id)
if self.options.unlock: ep.locked = None
if ep.locked and not self.options.force:
if self.options.verbose:
print('#%s: "%s" locked on %s by %s' %
(ep.id, ep.name, ep.locked, ep.locked_by))
ret = None
else:
# ready_state:
# None means "don't care",
# ready == X means ready to do X,
# force is dangerous and will likely mess tings up.
if self.ready_state is None \
or ep.state==self.ready_state \
or self.options.force:
self.set_dirs(ep.show)
self.episode_dir = os.path.join(
self.show_dir, 'dv', ep.location.slug)
if self.options.verbose: print(ep.name)
if self.options.lag:
if sleepytime:
# don't lag on the first one that needs processing,
# we only want to lag between the fence posts.
print("lagging....", self.options.lag)
time.sleep(self.options.lag)
else:
sleepytime = True
self.log_in(ep)
if not self.options.quiet:
print(self.__class__.__name__, ep.id, ep.name)
if self.options.skip:
ret = True
else:
ret = self.process_ep(ep)
if self.options.verbose: print("process_ep:", ret)
# .process is long running (maybe, like encode or post)
# so refresh episode in case its .stop was set
# (would be set in some other process, like the UI)
ep = Episode.objects.get(pk=e.id)
if ret:
# if the process doesn't fail,
# and it was part of the normal process,
# don't bump if the process was forced,
# even if it would have been had it not been forced.
# if you force, you know better than the process,
# so the process is going to let you bump.
# huh?!
# so.. ummm...
# 1. you can't bump None
# 2. don't bump when in test mode
# 3. if it wasn't forced:, bump.
if self.ready_state is not None \
and not self.options.test \
and not self.options.force:
# bump state
ep.state += 1
self.end = datetime.datetime.now()
ep.save()
self.log_out(ep)
if ep.stop:
if self.options.verbose:
print(".STOP set on the episode.")
# send message to .process_eps which bubbles up to .poll
self.stop = True
# re-set the stop flag.
ep.stop = False
ep.save()
break
else:
if self.options.verbose:
print('#%s: "%s" is in state %s, ready is %s' %
(ep.id, ep.name, ep.state, self.ready_state))
return ret
def one_show(self, show):
"""
"""
self.set_dirs(show)
locs = Location.objects.filter(show=show)
if self.options.room:
locs = locs.filter(slug=self.options.room)
for loc in locs:
if self.options.verbose: print(loc.name)
# episodes = Episode.objects.filter( location=loc).order_by(
# 'sequence','start',)
# if self.options.day:
# episodes=episodes.filter(start__day=self.options.day)
self.one_loc(show, loc)
def work(self):
"""
find and process episodes
"""
episodes = Episode.objects
if self.options.client:
clients = Client.objects.filter(slug=self.options.client)
episodes = episodes.filter(show__client__in=clients)
if self.options.show:
shows = Show.objects.filter(slug=self.options.show)
episodes = episodes.filter(show__in=shows)
if self.options.day:
episodes = episodes.filter(start__day=self.options.day)
if self.options.room:
episodes = episodes.filter(location__slug=self.options.room)
if self.args:
episodes = episodes.filter(id__in=self.args)
if self.ready_state is not None \
and not self.options.force:
episodes = episodes.filter(state=self.ready_state)
# episodes = Episode.objects.order_by( 'sequence','start',)
self.start = datetime.datetime.now()
ret = self.process_eps(episodes)
self.end = datetime.datetime.now()
work_time = self.end - self.start
if work_time.seconds or True:
print("run time: %s minutes" % (work_time.seconds / 60))
return ret
def poll(self):
done = False
while not done:
ret = self.work()
if self.stop:
done = True
else:
if self.options.verbose: print("sleeping....")
time.sleep(int(self.options.poll))
return ret
def list(self):
"""
list clients and shows.
"""
clients=Client.objects.annotate(
max_date=Max('show__episode__start')) \
.filter(active=True) \
.order_by('max_date')
if self.options.client:
clients = clients.filter(slug=self.options.client)
for client in clients:
print("\nName: %s Slug: %s" % (client.name, client.slug))
shows=Show.objects.filter(client=client)\
.annotate( max_date=Max('episode__start'))\
.order_by('max_date')
# if self.options.show:
# shows = shows.filter(slug=self.options.show)
for show in shows:
print(show)
print("\tName: %s Slug: %s" % (show.name, show.slug))
print("\t--client %s --show %s" % (client.slug, show.slug))
print("client=%s\nshow=%s" % (client.slug, show.slug))
locations = show.locations.filter(
active=True).order_by('sequence')
for loc in locations:
print(("room={}".format(loc.slug)))
if self.options.verbose:
for ep in Episode.objects.filter(show=show):
print("\t\t id: %s state: %s %s" %
(ep.id, ep.state, ep))
return
def add_more_options(self, parser):
pass
def add_more_option_defaults(self, parser):
pass
def set_options(self, *bar, **extra_options):
# hook for test runner
self.extra_options = extra_options
def get_options(self):
for k, v in self.extra_options.items():
self.options.ensure_value(k, v)
setattr(self.options, k, v)
def parse_args(self):
"""
parse command line arguments.
"""
parser = optparse.OptionParser()
# hardcoded defaults
# parser.set_defaults(dv_format="square_ntsc_wide")
parser.set_defaults(dv_format="hdv_720_30p")
parser.set_defaults(upload_formats="mp4")
parser.set_defaults(media_dir=os.path.expanduser('~/Videos/veyepar'))
parser.set_defaults(lag=0)
self.add_more_option_defaults(parser)
# read from config file, overrides hardcoded
"""
>>> open('blip_uploader.cfg').read()
'[global]\ncategory=7\nlicense=13\ntopics=test\n'
>>> config.read('blip_uploader.cfg')
>>> config.items('global')
[('category', '7'), ('topics', 'test'), ('license', '13')]
"""
config = configparser.RawConfigParser()
files = config.read(
['veyepar.cfg', os.path.expanduser('~/veyepar.cfg')])
if files:
d = dict(config.items('global'))
d['whack'] = False # don't want this somehow getting set in .config - too dangerous.
parser.set_defaults(**d)
if 'verbose' in d:
print("using config file(s):", files)
print(d)
parser.add_option(
'-m',
'--media-dir',
help="media files dir",
)
parser.add_option(
'--tmp-dir',
help="temp files dir (should be local)",
)
parser.add_option('-c', '--client')
parser.add_option('-s', '--show')
parser.add_option('-d', '--day')
parser.add_option('-r', '--room', help="Location")
parser.add_option('--dv-format', help='pal, pal_wide, ntsc, ntsc_wide')
parser.add_option('--upload-formats', help='ogg, ogv, mp4, flv, dv')
parser.add_option('-l', '--list', action="store_true")
parser.add_option('-v', '--verbose', action="store_true")
parser.add_option('-q', '--quiet', action="store_true")
parser.add_option(
'--debug-log',
action="store_true",
help="append logs to .description so it gets posted to blip")
parser.add_option(
'--test',
action="store_true",
help="test mode - do not make changes to the db. maybe. "
"(not fully implemented, for development use.")
parser.add_option('--ready-state',
type="int",
help="set self.ready_state")
parser.add_option('--force',
action="store_true",
help="override ready state and lock, use with care.")
parser.add_option('--unlock',
action="store_true",
help="clear locked status, use with care.")
parser.add_option('--whack',
action="store_true",
help="whack current episodes, use with care.")
parser.add_option('--skip',
action="store_true",
help="skip processing and bump state.")
parser.add_option('--lag',
type="int",
help="delay in seconds between processing episodes.")
parser.add_option('--poll', help="poll every x seconds.")
self.add_more_options(parser)
self.options, self.args = parser.parse_args()
# this needs to be done better:
self.options.upload_formats = self.options.upload_formats.split()
self.get_options()
if self.options.verbose:
print(self.options, self.args)
if "pal" in self.options.dv_format:
self.fps = 25.0
self.bpf = 144000
if self.options.ready_state:
self.ready_state = self.options.ready_state
return
def main(self):
self.parse_args()
if self.options.list:
ret = self.list()
elif self.options.poll:
ret = self.poll()
else:
ret = self.work()
return ret
class process(object):
"""
Abstract class for processing.
Provides basic options and itarators.
Only operates on episodes in ready_state,
or all if ready_state is None.
if read_sate is not None and .processing() returns True,
bump the episode's state:
state=ready_state+1
"""
extra_options={}
# set stop to True to stop processing at the end of episode
# will not abort a process, like an encode, upload or anything like that.
# it just exits these loops:
# for e in es: (process episodes) and do while not Done: (poll)
stop = False
ready_state=None
# defaults to ntsc stuff
# fps=29.98
fps=30000/1001.0
bpf=120000
def run_cmd(self,cmd):
"""
given command list
if verbose prints stuff,
runs it, returns pass/fail.
"""
if self.options.verbose:
print cmd
print ' '.join(cmd)
if self.options.test:
print "TEST: not running command."
return True
p=subprocess.Popen(cmd)
p.wait()
retcode=p.returncode
if retcode:
if self.options.verbose:
print "command failed"
print retcode
ret = False
else:
ret = True
return ret
def run_cmds(self, episode, cmds):
"""
given a list of commands
append them to the episode's shell script
then run each
abort and return False if any fail.
"""
script_pathname = os.path.join(self.work_dir, episode.slug+".sh")
sh = open(script_pathname,'a')
for cmd in cmds:
if type(cmd)==list:
print cmd
log_text = ' '.join(cmd)
else:
log_text = cmd
cmd=cmd.split()
sh.writelines([t.encode('utf-8','ignore') for t in log_text])
# sh.writelines(log_text)
# if isinstance(s, unicode): return s.encode('utf-8') else: return s
sh.write('\n')
# self.log_info(log_text)
if self.options.debug_log:
episode.description += "\n{0:>s}\n".format(log_text, )
episode.save()
if not self.run_cmd(cmd):
return False
sh.write('\n')
sh.close()
return True
def set_dirs(self,show):
client = show.client
self.show_dir = os.path.join(
self.options.media_dir,client.slug,show.slug)
if self.options.tmp_dir:
self.tmp_dir = self.options.tmp_dir
else:
self.tmp_dir = os.path.join(self.show_dir,"tmp")
# change me to "cmd" or something before the next show
self.work_dir = os.path.join(self.show_dir,"tmp")
return
def log_in(self,episode):
"""
log_in/out
create logs, and lock.unlock the episode
"""
where=socket.gethostname()
if os.environ.get('STY') is not None:
where += ":" + os.environ['STY']
what_where = "%s@%s" % (self.__class__.__name__, where)
episode.locked = datetime.datetime.now()
episode.locked_by = what_where
# save the lock to disk
episode.save()
try:
state = State.objects.get(id=episode.state)
except State.DoesNotExist:
state = None
self.start=datetime.datetime.now()
self.log=Log(episode=episode,
state=state,
ready = datetime.datetime.now(),
start = datetime.datetime.now(),
result = what_where)
self.log.save()
def log_info(self,text):
self.log.result += text + '\n'
def log_out(self, episode):
# done processing
# log the end time
# unlock the episode
self.log.end = datetime.datetime.now()
self.log.save()
episode.locked = None
episode.locked_by = ''
episode.save()
def process_ep(self, episode):
print "stubby process_ep", episode.id, episode.name
return
def process_eps(self, episodes):
ret = None
sleepytime=False
for e in episodes:
# next line requires the db to make sure the lock field is fresh
ep=Episode.objects.get(pk=e.id)
if self.options.unlock: ep.locked=None
if ep.locked and not self.options.force:
if self.options.verbose:
print '#%s: "%s" locked on %s by %s' % (ep.id, ep.name, ep.locked, ep.locked_by)
ret = None
else:
# None means "don't care",
# ready means ready,
# force is dangerous and will likely mess tings up.
if self.ready_state is None \
or ep.state==self.ready_state \
or self.options.force:
self.set_dirs(ep.show)
self.episode_dir=os.path.join(
self.show_dir, 'dv', ep.location.slug )
if self.options.verbose: print ep.name
if self.options.lag:
if sleepytime:
# don't lag on the first one that needs processing,
# we only want to lag between the fence posts.
print "lagging....", self.options.lag
time.sleep(self.options.lag)
else:
sleepytime = True
self.log_in(ep)
if not self.options.quiet: print self.__class__.__name__, ep.id, ep.name
if self.options.skip:
ret = True
else:
ret = self.process_ep(ep)
if self.options.verbose: print "process_ep:", ret
# .process is long running (maybe, like encode or post)
# so refresh episode in case its .stop was set
# (would be set in some other process, like the UI)
ep=Episode.objects.get(pk=e.id)
if ret:
# if the process doesn't fail,
# and it was part of the normal process,
# don't bump if the process was forced,
# even if it would have been had it not been forced.
# if you force, you know better than the process,
# so the process is going to let you bump.
# huh?!
# so.. ummm...
# 1. you can't bump None
# 2. don't bump when in test mode
# 3. if it wasn't forced:, bump.
if self.ready_state is not None \
and not self.options.test \
and not self.options.force:
# bump state
ep.state += 1
self.end = datetime.datetime.now()
ep.save()
self.log_out(ep)
if ep.stop:
if self.options.verbose: print ".STOP set on the episode."
# send message to .process_eps which bubbles up to .poll
self.stop = True
# re-set the stop flag.
ep.stop = False
ep.save()
break
else:
if self.options.verbose:
print '#%s: "%s" is in state %s, ready is %s' % (
ep.id, ep.name, ep.state, self.ready_state)
return ret
def one_show(self, show):
"""
"""
self.set_dirs(show)
locs = Location.objects.filter(show=show)
if self.options.room:
loc=Location.objects.get(name=self.options.room)
locs=locs.filter(location=loc)
for loc in locs:
if self.options.verbose: print loc.name
episodes = Episode.objects.filter( location=loc).order_by(
'sequence','start',)
if self.options.day:
episodes=episodes.filter(start__day=self.options.day)
for day in [17,18,19,20,21]:
# self.process_eps(episodes.filter(start__day=day))
es=episodes.filter(start__day=day)
self.process_eps(es)
def work(self):
"""
find and process episodes
"""
episodes = Episode.objects
if self.options.client:
clients = Client.objects.filter(slug=self.options.client)
episodes = episodes.filter(show__client__in=clients)
if self.options.show:
shows = Show.objects.filter(slug=self.options.show)
episodes = episodes.filter(show__in=shows)
if self.options.day:
episodes = episodes.filter(start__day=self.options.day)
if self.options.room:
episodes = episodes.filter(location__slug=self.options.room)
if self.args:
episodes = episodes.filter(id__in=self.args)
self.start=datetime.datetime.now()
ret = self.process_eps(episodes)
self.end=datetime.datetime.now()
work_time = self.end-self.start
if work_time.seconds or True:
print "run time: %s minutes" % (work_time.seconds/60)
return ret
def poll(self):
done=False
while not done:
ret = self.work()
if self.stop:
done=True
else:
if self.options.verbose: print "sleeping...."
time.sleep(int(self.options.poll))
return ret
def list(self):
"""
list clients and shows.
"""
for client in Client.objects.all():
print "\nName: %s Slug: %s" %( client.name, client.slug )
for show in Show.objects.filter(client=client):
print "\tName: %s Slug: %s" %( show.name, show.slug )
print "\t--client %s --show %s" %( client.slug, show.slug )
print "client=%s\nshow=%s" %( client.slug, show.slug )
if self.options.verbose:
for ep in Episode.objects.filter(show=show):
print "\t\t id: %s state: %s %s" % (
ep.id, ep.state, ep )
return
def add_more_options(self, parser):
pass
def add_more_option_defaults(self, parser):
pass
def set_options(self,*bar,**extra_options):
# hook for test runner
self.extra_options=extra_options
def get_options(self):
for k,v in self.extra_options.iteritems():
self.options.ensure_value(k,v)
setattr(self.options, k, v)
def parse_args(self):
"""
parse command line arguments.
"""
parser = optparse.OptionParser()
# hardcoded defaults
parser.set_defaults(dv_format="ntsc")
parser.set_defaults(upload_formats="mp4")
parser.set_defaults(media_dir=os.path.expanduser('~/Videos/veyepar'))
parser.set_defaults(lag=0)
self.add_more_option_defaults(parser)
# read from config file, overrides hardcoded
"""
>>> open('blip_uploader.cfg').read()
'[global]\ncategory=7\nlicense=13\ntopics=test\n'
>>> config.read('blip_uploader.cfg')
>>> config.items('global')
[('category', '7'), ('topics', 'test'), ('license', '13')]
"""
config = ConfigParser.RawConfigParser()
files=config.read(['veyepar.cfg',
os.path.expanduser('~/veyepar.cfg')])
if files:
d=dict(config.items('global'))
d['whack']=False # don't want this somehow getting set in .config - too dangerous.
parser.set_defaults(**d)
if 'verbose' in d:
print "using config file(s):", files
print d
parser.add_option('-m', '--media-dir',
help="media files dir", )
parser.add_option('--tmp-dir',
help="temp files dir (should be local)", )
parser.add_option('-c', '--client' )
parser.add_option('-s', '--show' )
parser.add_option('-d', '--day' )
parser.add_option('-r', '--room',
help="Location")
parser.add_option('--dv-format',
help='pal, pal_wide, ntsc, ntsc_wide' )
parser.add_option('--upload-formats',
help='ogg, ogv, mp4, flv, dv' )
parser.add_option('-l', '--list', action="store_true" )
parser.add_option('-v', '--verbose', action="store_true" )
parser.add_option('-q', '--quiet', action="store_true" )
parser.add_option('--debug-log', action="store_true",
help="append logs to .description so it gets posted to blip" )
parser.add_option('--test', action="store_true",
help="test mode - do not make changes to the db. maybe. "
"(not fully implemented, for development use.")
parser.add_option('--ready-state', type="int",
help="set self.ready_state" )
parser.add_option('--force', action="store_true",
help="override ready state and lock, use with care." )
parser.add_option('--unlock', action="store_true",
help="clear locked status, use with care." )
parser.add_option('--whack', action="store_true",
help="whack current episodes, use with care." )
parser.add_option('--skip', action="store_true",
help="skip processing and bump state." )
parser.add_option('--lag', type="int",
help="delay in seconds between processing episodes.")
parser.add_option('--poll',
help="poll every x seconds." )
self.add_more_options(parser)
self.options, self.args = parser.parse_args()
# this needs to be done better:
self.options.upload_formats = self.options.upload_formats.split()
self.get_options()
if self.options.verbose:
print self.options, self.args
if "pal" in self.options.dv_format:
self.fps=25.0
self.bpf=144000
if self.options.ready_state:
self.ready_state = self.options.ready_state
return
def main(self):
self.parse_args()
if self.options.list:
ret = self.list()
elif self.options.poll:
ret = self.poll()
else:
ret = self.work()
return ret
class process():
"""
Abstract class for processing.
Provides basic options and itarators.
Only operates on episodes in ready_state,
or all if ready_state is None.
if read_sate is not None and .processing() returns True,
bump the episode's state:
state=ready_state+1
"""
extra_options = {}
# set stop to True to stop processing at the end of episode
# will not abort a process, like an encode, upload or anything like that.
# it just exits these loops:
# for e in es: (process episodes) and do while not Done: (poll)
stop = False
ready_state = None
# defaults to ntsc stuff
# fps=29.98
fps = 30000 / 1001.0
bpf = 120000
def save_me(self, o):
# tring to fix the db timeout problem
try:
o.save()
except DatabaseError as e:
connection.connection.close()
connection.connection = None
o.save()
def run_cmd(self, cmd):
"""
given command list
if verbose prints stuff,
runs it, returns pass/fail.
"""
if self.options.verbose:
print(cmd)
print(' '.join(cmd))
if self.options.test:
print("TEST: not running command.")
return True
p = subprocess.Popen(cmd)
p.wait()
retcode = p.returncode
if retcode:
if self.options.verbose:
print("command failed")
print(retcode)
ret = False
else:
ret = True
return ret
def run_cmds(self, episode, cmds):
"""
given a list of commands
append them to the episode's shell script
then run each
abort and return False if any fail.
"""
script_pathname = os.path.join(self.work_dir, episode.slug+".sh")
sh = open(script_pathname, 'a')
for cmd in cmds:
if type(cmd) == list:
print(cmd)
log_text = ' '.join(cmd)
else:
log_text = cmd
cmd = cmd.split()
try:
# Not sure what python v3 wants. str I guess.
# .write(b'a') and .writeline(b'a') - Errpr must be str, not b
sh.writelines(log_text)
# sh.writelines([t.encode('utf-8','ignore') for t in log_text])
except Exception as e:
import code; code.interact(local=locals())
import sys; sys.exit()
# if isinstance(s, unicode): return s.encode('utf-8') else: return s
sh.write('\n')
# self.log_info(log_text)
if self.options.debug_log:
episode.description += "\n{0:>s}\n".format(log_text, )
episode.save()
if not self.run_cmd(cmd):
return False
sh.write('\n')
sh.close()
return True
def file2cdn(self, show, src, dst=None):
"""
src is relitive to the show dir.
src and dst get filled to full paths.
Check to see if src exists,
if it does, try to upload it to cdn
(rax_uploader will skip if same file exists).
"""
print("checking:", src, end='')
if dst is None:
dst = src
src = os.path.join(self.show_dir, src)
dst = os.path.join("veyepar", show.client.slug, show.slug, dst)
if os.path.exists(src):
u = rax_uploader.Uploader()
u.user = show.client.rax_id
if not show.client.bucket_id.strip():
raise AttributeError("client.bucket_id is blank")
u.bucket_id = show.client.bucket_id
u.bucket = show.client.bucket_id
u.pathname = src
u.key_id = dst
print(" uploading....")
ret = u.upload()
print(u.new_url)
ret = u.new_url
else:
print(("file2cdn can't find {}".format(src)))
ret = False
return ret
def set_dirs(self, show):
client = show.client
self.show_dir = os.path.join(
self.options.media_dir, client.slug, show.slug)
if self.options.tmp_dir:
self.tmp_dir = self.options.tmp_dir
else:
self.tmp_dir = os.path.join(self.show_dir, "tmp")
# change me to "cmd" or something before the next show
self.work_dir = os.path.join(self.show_dir, "tmp")
return
def log_in(self, episode):
"""
log_in/out
create logs, and lock.unlock the episode
"""
where = socket.gethostname()
if os.environ.get('STY') is not None:
where += ":" + os.environ['STY']
what_where = "%s@%s" % (self.__class__.__name__, where)
episode.locked = datetime.datetime.now()
episode.locked_by = what_where[:35]
# save the lock to disk
episode.save()
try:
state = State.objects.get(sequence=episode.state)
except State.DoesNotExist:
state = None
self.start = datetime.datetime.now()
self.log = Log(
episode=episode,
state=state,
ready=datetime.datetime.now(),
start=datetime.datetime.now(),
result=what_where)
self.log.save()
def log_info(self, text):
self.log.result += text + '\n'
def log_out(self, episode):
# done processing
# log the end time
# unlock the episode
self.log.end = datetime.datetime.now()
self.log.save()
episode.locked = None
episode.locked_by = ''
episode.save()
def process_ep(self, episode):
print("stubby process_ep: #{} state:{} {}".format(
episode.id, episode.state, episode.name))
return
def ep_in_shard(self, episode):
"""shard over ...umm.. a bunch of tmux sessions I think"""
s = os.environ.get('STY')
if s:
s = s.split('-')
if len(s) == 3:
s = s[-1]
if s.isdigit():
return episode.id % 11 == int(s)
return True
def ep_is_available(self, episode):
"""Is episode something we want to act on? Return boolean.
Will take a lock on the episode.
"""
e_id = episode.id
if not self.ep_in_shard(episode):
return False
with transaction.atomic():
# require the db to make sure the lock field is fresh
try:
ep = Episode.objects.select_for_update(skip_locked=True) \
.get(pk=e_id)
except Episode.DoesNotExist:
print('#%s: "%s" is locked at a DB level'
% (e_id, episode.name))
return False
# ready_state:
# None means "don't care",
# ready == X means ready to do X,
# force is dangerous and will likely mess tings up.
if self.ready_state is not None \
and ep.state != self.ready_state \
and not self.options.force:
if self.options.verbose:
print('#%s: "%s" is in state %s, ready is %s'
% (e_id, ep.name, ep.state, self.ready_state))
return False
if self.options.unlock:
ep.locked = None
if ep.locked and not self.options.force:
if self.options.verbose:
print('#%s: "%s" locked on %s by %s'
% (e_id, ep.name, ep.locked, ep.locked_by))
return False
if not self.options.test and not self.options.skip:
# Don't log or lock when testing
self.log_in(ep)
return True
def process_eps(self, episodes):
# if self.options.verbose: print("process_ep: enter")
ret = None
for e in episodes:
if not self.ep_is_available(e):
continue
# It may have been a while since whe retrieved it
# And we've just taken a lock in a separate transaction
e.refresh_from_db()
self.set_dirs(e.show)
self.episode_dir = os.path.join(
self.show_dir, 'dv', e.location.slug)
if self.options.verbose:
print(e.name)
if not self.options.quiet:
print(self.__class__.__name__, e.id, e.name)
if not self.options.skip:
ret = self.process_ep(e)
if self.options.verbose:
print("process_ep:", ret)
# .process is long running (maybe, like encode or post)
# so refresh episode in case its .stop was set
# (would be set in some other process, like the UI)
try:
e_id = e.id
e = Episode.objects.get(pk=e_id)
except DatabaseError as err:
connection.connection.close()
connection.connection = None
e = Episode.objects.get(pk=e_id)
if ret:
# if the process doesn't fail,
# and it was part of the normal process,
# don't bump if the process was forced,
# even if it would have been had it not been forced.
# if you force, you know better than the process,
# so the process is going to let you bump.
# huh?!
# so.. ummm...
# 1. you can't bump None
# 2. don't bump when in test mode
# 3. if it wasn't forced:, bump.
if self.ready_state is not None \
and not self.options.test \
and not self.options.force:
# bump state
e.state += 1
self.end = datetime.datetime.now()
e.save()
if not self.options.test:
self.log_out(e)
if e.stop:
if self.options.verbose:
print(".STOP set on the episode.")
# send message to .process_eps which bubbles up to .poll
self.stop = True
# re-set the stop flag.
e.stop = False
e.save()
break
if self.options.lag:
print("lagging....", self.options.lag)
time.sleep(self.options.lag)
return ret
def one_show(self, show):
"""
"""
self.set_dirs(show)
locs = Location.objects.filter(show=show)
if self.options.room:
locs = locs.filter(slug=self.options.room)
for loc in locs:
if self.options.verbose:
print(loc.name)
# episodes = Episode.objects.filter( location=loc).order_by(
# 'sequence','start',)
# if self.options.day:
# episodes=episodes.filter(start__day=self.options.day)
self.one_loc(show, loc)
def work(self):
"""
find and process episodes
"""
# if self.options.verbose: print("work: enter")
episodes = Episode.objects
if self.options.client:
clients = Client.objects.filter(slug=self.options.client)
episodes = episodes.filter(show__client__in=clients)
if self.options.show:
shows = Show.objects.filter(slug=self.options.show)
episodes = episodes.filter(show__in=shows)
if self.options.day:
episodes = episodes.filter(start__day=self.options.day)
if self.options.room:
episodes = episodes.filter(location__slug=self.options.room)
if self.args:
episodes = episodes.filter(id__in=self.args)
if self.options.resume:
episodes.exclude(log__id__gte=self.options.resume)
"""
log = Log.objects.get(id=self.options.resume)
start = log.start
print(start)
episodes = episodes.annotate( max_start=Max('log__start'))
episodes.exclude(max_start__gte=start)
"""
if self.ready_state is not None \
and not self.options.force:
episodes = episodes.filter(state=self.ready_state)
# episodes = Episode.objects.order_by( 'sequence','start',)
if episodes:
self.start = datetime.datetime.now()
ret = self.process_eps(episodes)
self.end = datetime.datetime.now()
work_time = self.end-self.start
if work_time.seconds or True:
print("run time: %s minutes" % (work_time.seconds/60))
else:
if self.options.poll == 0:
print("queue empty, poll 0, exiting.")
ret = False
else:
ret = True
return ret
def poll(self):
done = False
while not done:
ret = self.work()
if self.stop:
done = True
else:
if self.options.verbose:
print("sleeping....")
time.sleep(int(self.options.poll))
return ret
def list(self):
"""
list clients and shows.
"""
clients = Client.objects \
.annotate(max_date=Max('show__episode__start')) \
.filter(active=True) \
.order_by('max_date') \
if self.options.client:
clients = clients.filter(slug=self.options.client)
for client in clients:
print("\nName: %s Slug: %s" % (client.name, client.slug))
shows = Show.objects.filter(client=client) \
.annotate(max_date=Max('episode__start')) \
.order_by('max_date')
# if self.options.show:
# shows = shows.filter(slug=self.options.show)
for show in shows:
print(show)
print("\tName: %s Slug: %s" % (show.name, show.slug))
print("\t--client %s --show %s" % (client.slug, show.slug))
print("client=%s\nshow=%s" % (client.slug, show.slug))
locations = show.locations.filter(
active=True).order_by('sequence')
for loc in locations:
print(("room={}".format(loc.slug)))
if self.options.verbose:
for ep in Episode.objects.filter(show=show):
print("\t\t id: %s state: %s %s"
% (ep.id, ep.state, ep))
return
def add_more_options(self, parser):
pass
def add_more_option_defaults(self, parser):
pass
def set_options(self, *bar, **extra_options):
# hook for test runner
self.extra_options = extra_options
def get_options(self):
for k, v in self.extra_options.items():
self.options.ensure_value(k, v)
setattr(self.options, k, v)
def parse_args(self):
"""
parse command line arguments.
"""
parser = optparse.OptionParser()
# hardcoded defaults
# parser.set_defaults(dv_format="square_ntsc_wide")
parser.set_defaults(dv_format="hdv_720_30p")
parser.set_defaults(upload_formats="mp4")
parser.set_defaults(media_dir=os.path.expanduser('~/Videos/veyepar'))
parser.set_defaults(lag=0)
self.add_more_option_defaults(parser)
# read from config file, overrides hardcoded
"""
>>> open('blip_uploader.cfg').read()
'[global]\ncategory=7\nlicense=13\ntopics=test\n'
>>> config.read('blip_uploader.cfg')
>>> config.items('global')
[('category', '7'), ('topics', 'test'), ('license', '13')]
"""
config = configparser.RawConfigParser()
files = config.read([
'/etc/veyepar/veyepar.cfg',
os.path.expanduser('~/veyepar.cfg'),
'veyepar.cfg',
])
if files:
d = dict(config.items('global'))
# don't want this somehow getting set in .config - too dangerous.
d['whack'] = False
parser.set_defaults(**d)
if 'verbose' in d:
print("using config file(s):", files)
print(d)
parser.add_option('-m', '--media-dir',
help="media files dir", )
parser.add_option('--tmp-dir',
help="temp files dir (should be local)", )
parser.add_option('-c', '--client')
parser.add_option('-s', '--show')
parser.add_option('-d', '--day')
parser.add_option('-r', '--room',
help="Location")
parser.add_option('--dv-format',
help='pal, pal_wide, ntsc, ntsc_wide')
parser.add_option('--upload-formats',
help='ogg, ogv, mp4, flv, dv')
parser.add_option('-l', '--list', action="store_true")
parser.add_option('-v', '--verbose', action="store_true")
parser.add_option('-q', '--quiet', action="store_true")
parser.add_option(
'--debug-log', action="store_true",
help="append logs to .description so it gets posted to blip")
parser.add_option(
'--test', action="store_true",
help="test mode - do not make changes to the db. maybe. "
"(not fully implemented, for development use.")
parser.add_option('--ready-state', type="int",
help="set self.ready_state")
parser.add_option('--force', action="store_true",
help="override ready state and lock, use with care.")
parser.add_option('--unlock', action="store_true",
help="clear locked status, use with care.")
parser.add_option('--whack', action="store_true",
help="whack current episodes, use with care.")
parser.add_option('--skip', action="store_true",
help="skip processing and bump state.")
parser.add_option('--resume', type="int",
help="resume a failed run.")
parser.add_option('--lag', type="int",
help="delay in seconds between processing episodes.")
parser.add_option('--poll',
help="poll every x seconds.")
self.add_more_options(parser)
self.options, self.args = parser.parse_args()
# strip # from #123
self.args = [a.strip('#') for a in self.args]
# this needs to be done better:
self.options.upload_formats = self.options.upload_formats.split()
self.get_options()
if self.options.verbose:
# import code; code.interact(local=locals())
pprint(self.options.__dict__)
pprint(self.args)
if "pal" in self.options.dv_format:
self.fps = 25.0
self.bpf = 144000
if self.options.ready_state:
self.ready_state = self.options.ready_state
return
def main(self):
self.parse_args()
if self.options.list:
ret = self.list()
elif self.options.poll:
ret = self.poll()
else:
ret = self.work()
return ret
def database_add_new_log(request):
constraint = helpers.constraint(request, "POST");
data = constraint.safe({
"contact_number2": False,
"purpose": False,
"type": True,
"visitor_id": True,
"document_from": False,
"document_type" : False
});
try:
of_user = UserProfile.objects.get(user=request.user.id);
of_person = Person.objects.get(id=data['visitor_id']);
log_details_params = {
"person_visitor": of_person,
"contact_number":data['contact_number2'],
"added_by" : of_user
};
if data['document_type']:
log_details_params['document_type'] = Document_type.objects.get(id=data['document_type']);
if data['document_from']:
log_details_params['document_from'] = Department.objects.get(id=data['document_from']);
of_log_details = Log_details(**log_details_params);
of_log_details.full_clean();
of_log_details.clean();
of_log_details.save();
of_log = Log(
type=data['type'],
purpose=data['purpose'],
details=of_log_details,
added_by=of_user
);
try:
of_log.full_clean();
of_log.clean();
of_log.save();
except e :
of_log_details.delete();
raise Exception(e);
pass;
return render(request, "ajax_temp/after_added_new_log.html", {
'log': of_log,
"obj_log" : Log,
"log_details_person" : of_log.details.person_visitor
});
except Person.DoesNotExist:
raise UserProfile.DoesNotExist("Index id of Person object does not exists");
except UserProfile.DoesNotExist:
raise UserProfile.DoesNotExist("Index id of UserProfile object does not exists");
except helpers.InvalidRequest:
return HttpResponse('Invalid Request');
pass;
def database_add_new_visitor(request):
constraint = helpers.constraint(request, "POST");
data = constraint.safe({
"contact_number": False,
"department": False,
"first_name": True,
"last_name": True,
"middle_name": False,
"gender": True,
"purpose": False,
"type": True,
"contact_number2": False,
"document_type": False,
"document_from": False
});
of_user = UserProfile.objects.get(user=request.user.id);
person_params = {
"first_name": data['first_name'],
"last_name": data['last_name'],
"middle_name": data['middle_name'],
"gender": data['gender'],
"added_by": of_user
};
if data['contact_number']:
person_params['contact_number'] = data['contact_number'];
pass;
if data['department']:
person_params['department'] = Department.objects.get(id=data['department']);
of_person = Person(**person_params);
of_person.full_clean();
of_person.clean();
of_person.save();
log_details_params = {
"person_visitor": of_person,
"contact_number":data['contact_number2'],
"added_by" : of_user
};
if data['document_type']:
log_details_params['document_type'] = Document_type.objects.get(id=data['document_type']);
if data['document_from']:
log_details_params['document_from'] = Department.objects.get(id=data['document_from']);
of_log_details = Log_details(**log_details_params);
of_log_details.full_clean();
of_log_details.clean();
of_log_details.save();
of_log = Log(
type=data['type'],
purpose=data['purpose'],
details=of_log_details,
added_by=of_user
);
try:
of_log.full_clean();
of_log.clean();
of_log.save();
except e :
of_log_details.delete();
raise Exception(e);
pass;
return render(request, "ajax_temp/after_added_new_log.html", {
'log': of_log,
"obj_log" : Log,
"log_details_person" : of_log.details.person_visitor
});
