def remove_tags(self, part_item):
SublerCLI = os.path.join(sys.path[0], "SublerCLI-v010")
filepath = part_item.modified_file_path()
#removal of artwork doesn't seem to work
all_tags = ["{Artwork:}", "{HD Video:}", "{Gapless:}", "{Content Rating:}", "{Media Kind:}", "{Name:}", "{Artist:}", "{Album Artist:}", "{Album:}", "{Grouping:}", "{Composer:}", "{Comments:}", "{Genre:}", "{Release Date:}", "{Track #:}", "{Disk #:}", "{TV Show:}", "{TV Episode #:}", "{TV Network:}", "{TV Episode ID:}", "{TV Season:}", "{Description:}", "{Long Description:}", "{Rating:}", "{Rating Annotation:}", "{Studio:}", "{Cast:}", "{Director:}", "{Codirector:}", "{Producers:}", "{Screenwriters:}", "{Lyrics:}", "{Copyright:}", "{Encoding Tool:}", "{Encoded By:}", "{contentID:}"]#these are currently not supported in subler cli tool, "{XID:}", "{iTunes Account:}", "{Sort Name:}", "{Sort Artist:}", "{Sort Album Artist:}", "{Sort Album:}", "{Sort Composer:}", "{Sort TV Show:}"]
logging.warning("removing tags...")
#Create the command line command
tag_removal_cmd = ['%s' % SublerCLI]
if self.opts.optimize:
action_description = "Tags removed and optimized"
tag_removal_cmd.append("-O")
else:
action_description = "Tags removed"
#end if optimize
tag_removal_cmd.append("-t")
tag_removal_cmd.append("".join(all_tags))
tag_removal_cmd.append("-i")
tag_removal_cmd.append(filepath)
success = self.execute_command(filepath, tag_removal_cmd, action_description)
if success:
Summary().metadata_removal_succeeded()
else:
Summary().metadata_removal_failed()
def tag(self, part_item):
SublerCLI = os.path.join(sys.path[0], "SublerCLI")
filepath = part_item.modified_file_path()
directory = os.path.dirname(filepath)
filename = os.path.basename(filepath)
filename_without_extension = os.path.splitext(filename)[0]
logging.warning("tagging...")
#Create the command line command
tag_cmd = ['%s' % SublerCLI]
if self.opts.optimize:
action_description = "Tags added and optimized"
tag_cmd.append("-optimize")
else:
action_description = "Tags added"
#end if optimize
if self.opts.embed_subtitles:
action_description += ", and any subtitles embedded"
compatible_sidecar_subtitles = self.get_all_sidecar_subtitles(
directory, filename_without_extension, codec='srt')
if len(compatible_sidecar_subtitles) == 0:
logging.warning("Found no 'srt' subtitle files to embed...")
else:
for sub in compatible_sidecar_subtitles:
tag_cmd.append("-s")
tag_cmd.append(os.path.join(directory, sub))
if self.opts.chapter_previews:
action_description += ", and chapter previews generated"
tag_cmd.append("-p")
tag_cmd.append("-metadata")
tag_cmd.append(part_item.tag_string()) #also downloads the artwork
tag_cmd.append("-source")
tag_cmd.append(filepath)
tag_cmd.append("-destination")
tmp_filepath = filepath + "_tmp"
tag_cmd.append(tmp_filepath)
success = self.execute_command(filepath, tag_cmd, action_description)
if success:
Summary().metadata_embedded_succeeded()
logging.info("Replacing old file with tagged one...")
shutil.move(tmp_filepath, filepath)
else:
Summary().metadata_embedded_failed()
logging.info("Cleaning up temporary file...")
os.unlink(tmp_filepath)
def update_model():
if memory.full:
replay = memory.sample(batch_size)
state_list = []
target_list = []
for sample in replay:
input = sample[0]
action = sample[1]
reward_list = sample[2]
possible_states = sample[3]
targetQ = []
_, allQ = sess.run([predict, Qout],
feed_dict={inputs: [input]})
if update_mode == "single":
next_state = possible_states[action]
next_input = normalize(next_state.grid_to_input())
Q1 = sess.run(Qout, feed_dict={inputs: [next_input]})
maxQ1 = np.max(Q1)
targetQ = allQ
targetQ[0, action] = reward_list[action] + \
(0 if next_state.halt else gamma * maxQ1)
elif update_mode == "all":
next_inputs = [
normalize(s.grid_to_input()) for s in possible_states
]
Q1 = sess.run(Qout, feed_dict={inputs: next_inputs})
maxQs = [np.max(Q) for Q in Q1]
targetQ = allQ
for k in range(4):
if possible_states[k].valid:
targetQ[0, k] = reward_list[k] + \
(0 if possible_states[k].halt else gamma * maxQs[k])
state_list.insert(0, input)
target_list.insert(0, targetQ[0])
_, summary = sess.run([updateModel, summary_op],
feed_dict={
inputs: state_list,
nextQ: target_list
})
Summary.write_summary_operation(summary, total_steps + steps)
def updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel = None):
if pComponent["componenttype"] == Cassette.componentType:
return Cassette.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Add.componentType:
return Add.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Evaporate.componentType:
return Evaporate.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Transfer.componentType:
return Transfer.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == React.componentType:
return React.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Prompt.componentType:
return Prompt.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Install.componentType:
return Install.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Comment.componentType:
return Comment.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == TrapF18.componentType:
return TrapF18.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == EluteF18.componentType:
return EluteF18.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Initialize.componentType:
return Initialize.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Mix.componentType:
return Mix.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Move.componentType:
return Move.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == Summary.componentType:
return Summary.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
elif pComponent["componenttype"] == ExternalAdd.componentType:
return ExternalAdd.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel)
else:
raise Exception("Unknown component type: " + pComponent["componenttype"])
def add_to_itunes(self, part_item):
if self.video_item.__class__.__name__ == "MovieItem":
itunes_playlist = "Movies"
else:
itunes_playlist = "TV Shows"
item_title = self.video_item.title
actionable_file_path = part_item.modified_file_path()
if not self.opts.force:
logging.warning("Finding '%s' in iTunes..." % actionable_file_path)
search_string = 'set currentItems to search playlist "%s" for "%s" only displayed' % (itunes_playlist, item_title)
delimiter = "###"
result_creation_string = 'set output to output & "%s"' % delimiter
does_item_exist_command = ["osascript", '-e', 'tell application "iTunes"', '-e', 'try' ,'-e', search_string, '-e', 'set output to ""', '-e', 'repeat with currentItem in currentItems', '-e', 'set loc to (location of currentItem)', '-e', 'if output is not equal to "" then', '-e', result_creation_string, '-e', 'end if', '-e', 'set output to output & POSIX path of loc', '-e', 'end repeat', '-e', 'output', '-e', 'end try', '-e', 'end tell']
logging.debug("'find in iTunes' script: %s" % (does_item_exist_command))
result_string = subprocess.Popen(does_item_exist_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn = self.preexec).communicate()[0].rstrip()
result_file_paths = result_string.split(delimiter)
logging.debug("Find results in iTunes: %s" % result_file_paths)
for file_path in result_file_paths:
unicode_filepath = normalize('NFC', file_path.decode('utf-8'))
if unicode_filepath == actionable_file_path:
logging.warning(" Already added to iTunes")
return
#end if result == actionable_file_path:
#end for file_path in result_file_paths
#end if not self.opts.force
logging.warning(" Adding to iTunes...")
file_str = 'set p to (POSIX file "%s")' % actionable_file_path
add_to_playlist_str = 'add p to playlist "%s"' % itunes_playlist
add_to_itunes_command = ['osascript', '-e', 'try', '-e', file_str, '-e', 'tell application "iTunes"', '-e', add_to_playlist_str, '-e', 'end tell', '-e', 'end try']
logging.debug("'add to iTunes' script: %s" % (add_to_itunes_command))
result = subprocess.Popen(add_to_itunes_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn = self.preexec).communicate()[0].rstrip()
if result.startswith("file track id"):
Summary().add_to_itunes_succeeded()
logging.warning(" Adding additional metadata to item in iTunes...")
current_item_str = 'tell %s' % result
set_rating_str = 'set rating to %i as integer' % self.video_item.itunes_rating()
set_play_count_str = 'set played count to %s as integer' % self.video_item.view_count
add_metadata_to_itunes_command = ['osascript', '-e', 'try', '-e', 'tell application "iTunes"', '-e', current_item_str, '-e', set_play_count_str, '-e', set_rating_str, '-e', 'end tell', '-e', 'end tell', '-e', 'end try']
logging.debug("'add additional metadata iTunes' script: %s" % (add_metadata_to_itunes_command))
subprocess.call(add_metadata_to_itunes_command)
else:
Summary().add_to_itunes_failed()
logging.critical("Failed 'add to iTunes' for '%s'. Incorrect path or not a compatible file type?" % (actionable_file_path) )
def optimize(self, part_item):
SublerCLI = os.path.join(sys.path[0], "SublerCLI-v010")
filepath = part_item.modified_file_path()
logging.warning("optimizing file...")
action_description = "Tags optimized"
#Create the command line command
optimize_cmd = ['%s' % SublerCLI]
optimize_cmd.append("-O")
optimize_cmd.append("-i")
optimize_cmd.append(filepath)
success = self.execute_command(filepath, optimize_cmd, action_description)
if success:
Summary().metadata_optimized_succeeded()
else:
Summary().metadata_optimized_failed()
def process(self):
skipped_all = True
if self.opts.gather_statistics:
skipped_all = False
LibraryStatistics().add_item(self.video_item)
#end if gather_statistics
for index, media_item in enumerate(self.media_items):
logging.warning("processing %d/%d media_items" %
(index + 1, len(self.media_items)))
part_items = media_item.part_items
for index2, part_item in enumerate(part_items):
no_action = True
logging.warning(" processing %d/%d part_items" %
(index2 + 1, len(part_items)))
Summary().increment_parts_processed()
if self.opts.removetags and self.canTag(part_item):
skipped_all = no_action = False
self.remove_tags(part_item)
#end if removetags
if self.opts.tag and self.canTag(part_item) and self.shouldTag(
part_item):
skipped_all = no_action = False
self.tag(part_item)
#end if tag
if self.opts.optimize and not self.opts.tag and not self.opts.removetags and self.canTag(
part_item):
#optimize is done together with tagging or removing, so only needs to be done here if it's the exclusive action
skipped_all = no_action = False
self.optimize(part_item)
#end if optimize
if self.opts.export_resources:
skipped_all = no_action = False
self.export_resources(part_item)
#end if export_resouces
if self.opts.add_to_itunes:
skipped_all = no_action = False
self.add_to_itunes(part_item)
#end if add_to_itunes
if self.opts.open_file_location and not no_action:
logging.warning("opening '%s'..." %
part_item.modified_file_path())
command = ['open', "-R", part_item.modified_file_path()]
subprocess.call(command)
#end if self.opts.open_file_location
#end for part_items
#end for media_items
if skipped_all:
logging.warning(
"skipping: found no suitable files for specified tasks")
#end if skipped_all
#end def process_item
#end MediaPartProcessor
def createFromComponent(nSequenceID,
pComponent,
username,
database,
systemModel=None):
if pComponent["componenttype"] == Cassette.componentType:
return Cassette.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Add.componentType:
return Add.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Evaporate.componentType:
return Evaporate.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Transfer.componentType:
return Transfer.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == React.componentType:
return React.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Prompt.componentType:
return Prompt.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Install.componentType:
return Install.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Comment.componentType:
return Comment.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == TrapF18.componentType:
return TrapF18.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == EluteF18.componentType:
return EluteF18.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Initialize.componentType:
return Initialize.createFromComponent(nSequenceID, pComponent,
username, database, systemModel)
elif pComponent["componenttype"] == Mix.componentType:
return Mix.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Move.componentType:
return Move.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == Summary.componentType:
return Summary.createFromComponent(nSequenceID, pComponent, username,
database, systemModel)
elif pComponent["componenttype"] == ExternalAdd.componentType:
return ExternalAdd.createFromComponent(nSequenceID, pComponent,
username, database, systemModel)
else:
raise Exception("Unknown component type: " +
pComponent["componenttype"])
def __init__(self, y, x_global, x_local, kernel, mType=0, offset=None, sigma2_v1=False, tol=1.0e-6, maxIter=200, y_name="", y_off_name="", x_name_glob=[], x_name_loc=[], fle_name="", summaryGLM=False):
"""
Initialize class
"""
semiGWR_Base.__init__(self, y, x_global, x_local, kernel, mType, offset, sigma2_v1, tol, maxIter)
self.y_name = y_name
self.x_name_glob = x_name_glob
self.x_name_loc = x_name_loc
if y_off_name is None:
y_off_name = " "
self.y_off_name = y_off_name
# add x names-global
n_xname = len(self.x_name_glob)
if n_xname < self.nVars_glob:
for i in range(n_xname,self.nVars_glob):
self.x_name_glob.append("name" + str(i))
# add x names-local
n_xname = len(self.x_name_loc)
if n_xname < self.nVars_loc:
for i in range(n_xname,self.nVars_loc-1):
self.x_name_loc.append("name" + str(i))
if len(self.x_name_loc) < self.nVars_loc:
self.x_name_loc.insert(0,'Intercept')
self.fle_name = fle_name
self.mName = semiGWR_Names[self.mType]
if summaryGLM:
x = np.hstack((x_global,x_local))
x_name = [ elem for elem in x_name_glob]
x_name.extend(x_name_loc)
#if mType == 0:
#self.OLS = OLS(y,x,sigma2_v1,y_name,x_name,fle_name)
#else:
self.GLM = GLM(y, x, mType, offset, sigma2_v1, tol, maxIter, y_name, y_off_name, x_name, fle_name)
Summary.semiGWR(self)
def __init__(self,
y,
x,
kernel,
mType=0,
offset=None,
sigma2_v1=False,
tol=1.0e-6,
maxIter=200,
y_name="",
y_off_name="",
x_name=[],
fle_name="",
summaryGLM=False):
"""
Initialize class
"""
GWGLM_Base.__init__(self, y, x, kernel, mType, offset, None, None,
None, sigma2_v1, tol, maxIter)
self.y_name = y_name
self.x_name = x_name
if y_off_name is None:
y_off_name = " "
self.y_off_name = y_off_name
# add x names
n_xname = len(self.x_name)
if n_xname < self.nVars:
for i in range(n_xname, self.nVars - 1):
self.x_name.append("name" + str(i))
n_xname = len(self.x_name)
if n_xname < self.nVars:
self.x_name.insert(0, 'Intercept')
self.fle_name = fle_name
self.mName = gwglm_Names[self.mType]
if summaryGLM:
#if mType == 0:
#self.OLS = OLS(y,x,sigma2_v1,y_name,x_name,fle_name)
#else:
self.GLM = GLM(y, x, mType, offset, sigma2_v1, tol, maxIter,
y_name, y_off_name, x_name, fle_name)
#if mType == 0:
#Summary.GWR(GWRMod=self)
#else:
Summary.GWGLM(self)
def __init__(self, parent):
wx.Notebook.__init__(self, parent, id=wx.ID_ANY, style=wx.BK_DEFAULT)
self.AddPage(CourtCase.CourtCase(self), "Court Case")
self.AddPage(Reference.NestedReferencePanel(self), "Reference")
self.AddPage(Official.OfficialPanel(self), "Official")
self.AddPage(Defendant.NestedDefendantPanel(self), "Defendant")
self.AddPage(Plaintiff.NestedPlaintiffPanel(self), "Plaintiff")
self.AddPage(Prosecutor.NestedProsecutorPanel(self), "Prosecutor")
self.AddPage(Witness.NestedWitnessPanel(self), "Witness")
self.AddPage(Charges.ChargesPanel(self), "Charges")
self.AddPage(Summary.SummaryPanel(self), "Summary")
self.AddPage(CaseNotes.CaseNotesPanel(self), "Case Notes")
self.AddPage(HealingNotes.HealingNotesPanel(self), "Healing Notes")
self.AddPage(FurtherNotes.FurtherNotesPanel(self),
"Further Research Notes")
def MiniBatchTrain(config, load_model=False):
# Set learning parameters
gamma = config["discount-factor"]
num_episodes = config["epochs"]
learning_rate = config["learning-rate"]
[eps_start, eps_stop, eps_steps] = config["epsilon-params"]
model_id = config["model-id"]
memory_size = config["replay-size"]
exploration = Exploration.getExplorationFromArgs(config["exploration"])
batch_size = config["batch-size"]
update_mode = config["update-mode"]
tensorboard_port = config["tensorboard"]
# Initialize TF Network and variables
Qout, inputs = Network.getNetworkFromArgs(config["architecture"])
Qmean = tf.reduce_mean(Qout)
Qmax = tf.reduce_max(Qout)
predict = tf.argmax(Qout, 1)
# Initialize TF output and optimizer
nextQ = tf.placeholder(shape=[None, 4], dtype=tf.float32)
loss = Losses.getLossFromArgs(config["loss"])(nextQ, Qout)
trainer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
updateModel = trainer.minimize(loss)
init = tf.global_variables_initializer()
# Initialize tensorboard summary
summary_op = Summary.init_summary_writer(model_name=model_id,
var_list=[("loss", loss),
("Qmean", Qmean),
("Qmax", Qmax)],
tb_port=tensorboard_port)
# Random action parameter
_epsilon = Gradients.Exponential(start=eps_start, stop=eps_stop)
def epsilon(step):
# Exponentially decreasing epsilon to 0.1 for first 25% epochs, constant value of 0.1 from there on
if step < num_episodes / (100.0 / eps_steps):
return _epsilon(step / (num_episodes / (100.0 / eps_steps)))
else:
return eps_stop
memory = ReplayMemory(memory_size)
def update_model():
if memory.full:
replay = memory.sample(batch_size)
state_list = []
target_list = []
for sample in replay:
input = sample[0]
action = sample[1]
reward_list = sample[2]
possible_states = sample[3]
targetQ = []
_, allQ = sess.run([predict, Qout],
feed_dict={inputs: [input]})
if update_mode == "single":
next_state = possible_states[action]
next_input = normalize(next_state.grid_to_input())
Q1 = sess.run(Qout, feed_dict={inputs: [next_input]})
maxQ1 = np.max(Q1)
targetQ = allQ
targetQ[0, action] = reward_list[action] + \
(0 if next_state.halt else gamma * maxQ1)
elif update_mode == "all":
next_inputs = [
normalize(s.grid_to_input()) for s in possible_states
]
Q1 = sess.run(Qout, feed_dict={inputs: next_inputs})
maxQs = [np.max(Q) for Q in Q1]
targetQ = allQ
for k in range(4):
if possible_states[k].valid:
targetQ[0, k] = reward_list[k] + \
(0 if possible_states[k].halt else gamma * maxQs[k])
state_list.insert(0, input)
target_list.insert(0, targetQ[0])
_, summary = sess.run([updateModel, summary_op],
feed_dict={
inputs: state_list,
nextQ: target_list
})
Summary.write_summary_operation(summary, total_steps + steps)
with tf.Session() as sess:
sess.run(init)
total_steps = 0
for i in range(num_episodes):
# Reset environment and get first new observation
state = Game.new_game(4)
reward_sum = 0
steps = 0
rand_steps = 0
invalid_steps = 0
# The Q-Network
while not state.halt:
s = normalize(state.grid_to_input())
steps += 1
if i == 0:
state.printstate()
print ""
# Choose an action by greedily (with e chance of random action) from the Q-network
a, allQ = sess.run([predict, Qout], feed_dict={inputs: [s]})
possible_states, action, ra, invalid_prediction = exploration(
a[0], allQ, i, epsilon, state)
if ra:
rand_steps += 1
if invalid_prediction:
invalid_steps += 1
reward_list = []
for k, nextstate in enumerate(possible_states):
r = reward(state, nextstate)
if r is not 0:
r = np.log2(nextstate.score - state.score) / 2.0
reward_list.insert(k, r)
reward_sum += reward_list[action]
# [normailzed input, action_taken, rewards for all action, all possible states]
memory.push([s, action, reward_list, possible_states])
# update step
update_model()
state = possible_states[action]
maxtile = max([max(state.grid[k]) for k in range(len(state.grid))])
stat = {
'max-tile': maxtile,
'score': state.score,
'steps': steps,
'r': reward_sum,
'rand-steps': "{0:.3f}".format(float(rand_steps) / steps)
}
total_steps += steps
Summary.write_scalar_summaries(
[
("steps", steps),
("epsilon", epsilon(i)),
("score", state.score),
("rand-steps", float(rand_steps) / steps),
("maxtile", maxtile),
# ("invalid-steps", steps)
],
i)
print i, "\t", stat
sess.close()
import Genetic as g
import Summary as s
sentences = [s.Simple_Sentence("hello there"), s.Simple_Sentence("am"), s.Simple_Sentence("A big boy now")]
target = [1, 0, 2]
pop = g.population(5)
for i in pop:
i.calc_rankings(sentences)
print i.weights
print i.rankings
print g.fitness(i, target)
print "----"
def run(self):
"""Thread entry point"""
sRunError = ""
self.userSourceIDs = True
log.debug("Running Sequence")
try:
# Main sequence run loop
log.debug("sourceSequence -> %s" % str(self.sourceSequence))
sMessage = "Run of sequence \"" + self.sourceSequence["metadata"][
"name"] + "\" started."
log.debug(sMessage)
self.messaging.broadcastMessage(sMessage)
nComponentCount = len(self.sourceSequence["components"])
nCurrentComponent = 0
while nCurrentComponent < nComponentCount:
# Get the next component
pSourceComponent = self.sourceSequence["components"][
nCurrentComponent]
# Skip components until we find our start component
self.sourceComponentID = pSourceComponent["id"]
if self.initializing and (self.startComponentID != 0) and (
self.sourceComponentID != self.startComponentID):
log.debug("Skipping unit operation (" +
pSourceComponent["componenttype"] + ")")
nCurrentComponent += 1
continue
# Update our initializing and running flags
self.initializing = False
self.running = True
# Ignore any previous summary component
if pSourceComponent["componenttype"] == Summary.componentType:
log.debug("Skipping unit operation (" +
pSourceComponent["componenttype"] + ")")
nCurrentComponent += 1
continue
# Create and run the next unit operation
log.debug("Starting unit operation " +
str(self.sourceComponentID) + " (" +
pSourceComponent["componenttype"] + ")")
pSourceUnitOperation = UnitOperations.createFromComponent(
self.sourceSequenceID, pSourceComponent, self.username,
self.sequenceManager.database, self.systemModel)
self.runComponentID = pSourceUnitOperation.copyComponent(
self.sourceSequenceID, self.runSequenceID)
pRunComponent = self.sequenceManager.GetComponent(
self.username, self.runComponentID, self.runSequenceID)
pRunUnitOperation = UnitOperations.createFromComponent(
self.runSequenceID, pRunComponent, self.username,
self.sequenceManager.database, self.systemModel)
pRunUnitOperation.setDaemon(True)
pRunUnitOperation.start()
self.systemModel.SetUnitOperation(pRunUnitOperation)
log.debug("Unit Op is alive")
# Wait until the operation completes or we receive an abort signal
while pRunUnitOperation.is_alive() and not self.runAborted:
log.debug("Unit Op check alive")
time.sleep(0.25)
self.systemModel.SetUnitOperation(None)
if self.runAborted:
log.error("Unit Op ABORTED!")
pRunUnitOperation.setAbort()
raise Exception("Run aborted")
# Check for unit operation error
log.debug("Unit check for error")
sError = pRunUnitOperation.getError()
if sError != "":
log.error("UnitOperation: %s" %
pRunUnitOperation.__class__.__name__)
log.error("Unit operation failed: " + sError)
raise Exception(sError)
log.debug("Prepare to update operation details")
# Update the unit operation details in the database
UnitOperations.updateToComponent(pRunUnitOperation,
self.runSequenceID,
pRunComponent, self.username,
self.sequenceManager.database,
self.systemModel)
log.debug("After updateToComponent")
self.sequenceManager.UpdateComponent(self.username,
self.runSequenceID,
self.runComponentID, None,
pRunComponent)
log.debug("After updateComponent")
log.debug("Completed unit operation (" +
pRunComponent["componenttype"] + ")")
self.sourceComponentID = 0
self.runComponentID = 0
# Check if the user paused the sequence for editing
if self.runWillPause:
# Pause until editing is complete
log.debug("Pausing run for editing")
self.runWillPause = False
self.runIsPaused = True
while self.runIsPaused and not self.runAborted:
time.sleep(0.25)
if self.runAborted:
raise Exception("Sequence aborted")
log.debug("Continuing paused run")
# Reload the sequence and make sure it is still valid
self.sourceSequence = self.sequenceManager.GetSequence(
self.username, self.sourceSequenceID)
if not self.sourceSequence["metadata"]["valid"]:
raise Exceptions(
"Edited sequence is no longer valid (" +
self.sourceSequenceID + ")")
# Scan until we find the unit operation we just executed
nComponentCount = len(self.sourceSequence["components"])
nCurrentComponent = 0
while nCurrentComponent < nComponentCount:
if self.sourceSequence["components"][
nCurrentComponent]["id"] == pSourceComponent[
"id"]:
break
nCurrentComponent += 1
if nCurrentComponent == nComponentCount:
raise Exception(
"Failed to find previous component in edited sequence"
)
# Advance to the next component
nCurrentComponent += 1
except Exception as ex:
log.error("Sequence run failed: " + str(ex))
log.error("Trace Back %s" % traceback.format_exc())
sRunError = str(ex)
# Add the Summary unit operation to the sequence
pSummaryComponent = Summary.createNewComponent(sRunError)
self.runComponentID = self.database.CreateComponent(
self.username, self.runSequenceID, pSummaryComponent["type"],
pSummaryComponent["note"], json.dumps(pSummaryComponent))
# Fully validate the run sequence
self.sequenceManager.ValidateSequenceFull(self.username,
self.runSequenceID)
# Switch to using the run IDs rather than the source because the summary unit operation only exists in the run sequence
self.userSourceIDs = False
# Instantiate and start the summary unit operation
log.info("Starting summary unit operation")
pSummaryComponent = self.sequenceManager.GetComponent(
self.username, self.runComponentID, self.runSequenceID)
pSummaryUnitOperation = UnitOperations.createFromComponent(
self.runSequenceID, pSummaryComponent, self.username,
self.sequenceManager.database, self.systemModel)
pSummaryUnitOperation.setDaemon(True)
pSummaryUnitOperation.start()
self.systemModel.SetUnitOperation(pSummaryUnitOperation)
# Broadcast the summary unit operation message
self.messaging.broadcastMessage(pSummaryComponent["message"])
# Wait until the operation completes
while pSummaryUnitOperation.is_alive():
time.sleep(0.25)
log.info("Summary unit operation complete")
self.runComponentID = 0
# Run complete
log.debug("Run stopped")
self.running = False
model_path = './model'
tf.reset_default_graph()
"""
Creates the master worker that maintains the master network.
We then initialize the workers array.
"""
global_episodes = tf.Variable(0, dtype=tf.int32, name='global_episodes', trainable=False)
total_frames = tf.Variable(0, dtype=tf.int32, name='total_frames', trainable=False)
learning_rate = tf.train.polynomial_decay(LEARNING_RATE, total_frames, MAX_ITERATION // 2,
LEARNING_RATE * 0.1)
with tf.device("/cpu:0"):
summary_writer = tf.summary.FileWriter("./train/" + SUMMARY_NAME + str(BETA[BETA_i]))
summary = Summary(summary_writer)
master_worker = Worker('global', env, GAMMA, learning_rate, global_episodes, total_frames, model_path, False,
False, num_workers, summary, BETA)
workers = []
for i in range(num_workers):
print (i)
workers.append(
Worker(i, env, GAMMA, learning_rate, global_episodes, total_frames, model_path, render, save_img,
num_workers, summary, BETA[BETA_i]))
"""
Initializes tensorflow variables
"""
with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=1)) as session:
saver = tf.train.Saver(max_to_keep=5)
if load:
def main():
signal.signal(signal.SIGINT, signal_handler)
root = logging.getLogger()
root.setLevel(logging.WARNING)
root.addHandler(ColorizingStreamHandler())
parser = OptionParser(usage="\
%prog [options]\n\
Example 1: %prog --tag\n\
Example 2: %prog --tag -b --username='******' --interactive-password\n\
\ttag everything in the library, and authenticate to Plex Home as user [email protected] with password being prompted for (password can also be supplied using the --password option)\n\
Example 3: %prog -bq --tag --remove-all-tags --optimize --export-subtitles --embed-subtitles -ip 192.168.0.2 --port 55400\n\
Example 4: %prog --subtitles -m 'D:\Movies' '/Volumes/Media/Movies' -m '\\' '/'\n\
Example 5: %prog -tb --batch-mediatype=movie --batch-breadcrumb='kids>cars'\n\
\tonly tag movies who are in a section containing the word 'kids' and movies who's name contains 'cars'\n\
Example 6: %prog -tb --batch-mediatype=show --batch-breadcrumb='>lost>season 1>pilot'\n\
\tonly tag tv episodes, matches all sections, show name contains lost, season 1, episode title contains 'pilot'\n\
Example 7: %prog -tb --batch-breadcrumb='tv>weeds>>goat'\n\
\tonly tag items who are in a section who's title contains 'tv', where the movie or show name contains 'weeds', any season and episode title contains 'goat' \n\
%prog -h for full list of options\n\n\
Filepaths to media items in PMS need to be the same as on machine that is running this script (can be worked around by using the -m option to modify the file path).\
")
parser.add_option( "-t", "--tag", action="store_true", dest="tag",
help="tag all compatible file types, and update any previously tagged files (if metadata in plex has changed)")
parser.add_option( "--tag-update", action="store_true", dest="tag_update",
help="update previously tagged files if the PMS entry has changed since last time (modification time)")
parser.add_option( "--tag-tv-prefer-season-artwork", action="store_true", dest="tag_prefer_season_artwork",
help="when tagging tv show episodes, the season artwork will be used instead of the episode thumbnail")
parser.add_option( "-r", "--remove-tags", action="store_true", dest="removetags",
help="remove all compatible tags from the files")
parser.add_option( "-f", "--force", action="store_true", dest="force",
help="ignore previous work and steam ahead with task (will re-tag previously tagged files, re-enters data into TV, etc.)")
parser.add_option( "-o", "--optimize", action="store_true", dest="optimize",
help="interleave the audio and video samples, and put the \"MooV\" atom at the beginning of the file")
parser.add_option( "--chapter-previews", action="store_true", dest="chapter_previews",
help="generate preview images for any chapter markers")
parser.add_option( "--export-subtitles", action="store_true", dest="export_subtitles",
help="export any subtitles to the same path as the video file")
parser.add_option( "--embed-subtitles", action="store_true", dest="embed_subtitles",
help="embed compatible files with a compatible \"sidecar\" subtitle file if present")
parser.add_option( "--export-artwork", action="store_true", dest="export_artwork",
help="export the artwork to the same path as the video file")
parser.add_option( "--stats", action="store_true", dest="gather_statistics",
help="gather \"interesting\" statistics about the items being processed")
parser.add_option( "-m", action="append", type="string", dest="path_modifications", nargs=2, metavar="<find> <replace>",
help="perform a find & replace operation on the pms' media file paths (useful if you are running the script on a different machine than the one who is hosting the pms, i.e. the mount paths are different). Supply multiple times to perform several different replacements (operations are performed in order supplied).")
parser.add_option( "--open", action="store_true", dest="open_file_location",
help="open a Finder window at the containing folder of the file just processed (Mac OS X only)")
parser.add_option( "--add-to-tv", action="store_true", dest="add_to_tv",
help="adds the item to TV if not already present")
parser.add_option( "-i", "--ip", action="store", dest="ip", type="string",
help="specify an alternate IP address that hosts a PMS to connect to (default is localhost)")
parser.add_option( "-p", "--port", action="store", dest="port", type="int",
help="specify an alternate port number to use when connecting to the PMS (default is 32400)")
parser.add_option( "--username", action="store", dest="username", type="string",
help="specify the username to use when authenticating with the PMS (default is no authentication)")
parser.add_option( "--password", action="store", dest="password", type="string",
help="specify the password to use when authenticating with the PMS (default is no authentication)")
parser.add_option( "--interactive-password", action="store_true", dest="interactive_password",
help="the password to use when authenticating with the PMS will be supplied interactively")
parser.add_option( "--interactive", action="store_true", dest="interactive",
help="interactivly select files to operate on [default]")
parser.add_option( "-b", "--batch", action="store_false", dest="interactive",
help="disable interactive mode. Requires no human intervention once launched, and will perform operations on all valid files")
parser.add_option( "--batch-mediatype", action="store", dest="batch_mediatype", type="choice", choices=["any", "movie", "show"], metavar="[movie/show]",
help="only specified media type will be processed")
parser.add_option( "--batch-breadcrumb", action="store", dest="batch_breadcrumb", type="string", metavar="breadcrumb",
help="only items matching the breadcrumb trail will be processed. Components seperated by '>' (case insensitive)")
parser.add_option( "-v", "--verbose", dest="verbose", action="callback",
callback=setLogLevel, help='increase verbosity (can be supplied 0-2 times)')
parser.add_option( "-q", "--quiet", action="store_true", dest="quiet",
help="ninja-like processing (can only be used when in batch mode)")
parser.add_option( "-d", "--dry-run", action="store_true", dest="dryrun",
help="pretend to do the job, but never actually change or export anything. Pretends that all tasks succeed. Useful for testing purposes")
parser.set_defaults( tag=False, tag_update=False, tag_prefer_season_artwork=False, remove_tags=False,
optimize=False, chapter_previews=False, embed_subtitles=False,
export_resources=False, export_subtitles=False, export_artwork=False,
gather_statistics=False, open_file_location=False, add_to_tv=False,
force_tagging=False, dryrun=False,
interactive=True, quiet=False, batch_mediatype="any", batch_breadcrumb="",
ip="localhost", port=32400, username="", password="", interactive_password=False,
path_modifications=[])
try:
opts, args = parser.parse_args()
except OptionValueError as e:
parser.error(e)
if opts.export_subtitles or opts.export_artwork:
opts.export_resources = True
if not opts.tag and not opts.removetags and not opts.optimize and not opts.export_resources and not opts.add_to_tv and not opts.gather_statistics:
parser.error("No task to perform. Our work here is done...")
if opts.tag_prefer_season_artwork and not opts.tag:
parser.error("Cannot prefer season artwork when not tagging...")
if opts.chapter_previews and not opts.tag:
parser.error("Cannot generate chapter previews when not tagging...")
if opts.embed_subtitles and not opts.tag:
parser.error("Cannot embed subtitles when not tagging...")
if opts.tag_update and not opts.tag:
parser.error("Cannot update tags when not tagging...")
if opts.interactive and ( opts.batch_mediatype != "any" or len(opts.batch_breadcrumb) > 0):
parser.error("Cannot use batch filtering options when batch mode is not active...")
if opts.interactive_password:
opts.password = getpass.getpass("Password $")
if (len(opts.username) > 0 and not len(opts.password) > 0) or (len(opts.password) > 0 and not len(opts.username) > 0):
parser.error("Must supply both username and password when using authentication to connect to PMS...")
if len(opts.batch_breadcrumb) > 0:
opts.batch_breadcrumb = opts.batch_breadcrumb.lower().split(">")
opts.batch_breadcrumb.reverse()
if opts.quiet:
root.setLevel(logging.ERROR)
if opts.interactive and not root.isEnabledFor(logging.INFO):
root.setLevel(logging.INFO)
if opts.dryrun:
logging.critical( "WARNING, RUNNING IN 'DRY RUN MODE'. NO ACTUAL CHANGES WILL BE MADE" )
elif opts.removetags:
logging.critical( "WARNING, TAGS WILL BE REMOVED PERMANENTLY" )
elif opts.force:
logging.critical( "FORCE MODE ENABLED. THIS WILL BYPASS ANY 'HAS THIS BEEN DONE BEFORE' CHECKS" )
logging.error( generate_centered_padded_string(" Plex Media Tagger Started ") )
if opts.gather_statistics:
statistics = LibraryStatistics()
summary = Summary()
request_handler = PmsRequestHandler()
request_handler.ip = opts.ip
request_handler.port = opts.port
request_handler.setup_opener(opts.username, opts.password)
global section_processor
section_processor = SectionProcessor(opts, request_handler)
logging.error( "Connecting to PMS at %s:%d" % (opts.ip, opts.port) )
sections_container = request_handler.get_sections_container()
media_container = sections_container.getroot()
title = media_container.attrib['title1']
section_elements = media_container.getchildren()
section_element_choice = '' #default is empty == all
if opts.interactive:
logging.info( "List of sections for %s" % title )
for index, section_element in enumerate(section_elements):
logging.info( "%d. %s" %(index, section_element.attrib['title']) )
#end for
if len(section_elements) == 0:
logging.error( "No sections found" )
else:
logging.warning( "empty input equals all" )
#ask user what sections should be processed
section_element_choice = raw_input("Section to process $")
if section_element_choice != '':
try:
section_element_choice = int(section_element_choice)
except ValueError, e:
logging.debug(e)
logging.critical( "'%s' is not a valid section number" % input )
sys.exit(1)
def addToSummary(self, summary: Summary, zone, month, year):
excelFileName = "Monthly Report "+zone.upper()+" " + \
"01-"+"{:0>2d}".format(int(month))+"-"+year+".xlsx"
monthFullName = datetime.datetime.now().strftime("%B")
fileName = 'User Data/' + monthFullName.upper(
) + "_" + year + "/" + excelFileName
print(fileName)
# return
workbook = load_workbook(filename=fileName)
activeSheet = workbook['Bill Summary']
# activeSheet = workbook.active
# inserting and formatting rows
noOfRows = len(summary.getRecordsList()) - 3
if noOfRows > 0:
activeSheet.insert_rows(11, noOfRows)
formattedrow = list(activeSheet.rows)[9]
print(formattedrow[0])
# unformattedrow = list(worksheet.rows)[9]
for row in range(noOfRows):
unformattedrow = list(activeSheet.rows)[10 + row]
print(unformattedrow[0])
for (c1, c2) in zip(formattedrow, unformattedrow):
c2.style = copy(c1.style)
c2.border = copy(c1.border)
c2.number_format = copy(c1.number_format)
c2.font = copy(c1.font)
c2.alignment = copy(c1.alignment)
c2.fill = copy(c1.fill)
# inserting data into summary
records = summary.getRecordsList()
totalRecords = len(summary.getRecordsList())
record = 0
fromRow = 10
toRow = fromRow + totalRecords
i = 0
for i in range(fromRow, toRow):
print(records[record][0])
activeSheet['A' + str(i)] = record + 1
activeSheet['B' + str(i)] = records[record][0]
activeSheet['C' + str(i)] = "[" + str(records[record][1]) + "]"
activeSheet['D' + str(i)] = records[record][2]
activeSheet['E' + str(i)] = records[record][3]
activeSheet['F' + str(i)] = records[record][4]
activeSheet['G'+str(i)] = records[record][3] + \
records[record][4]
activeSheet['H' + str(i)] = records[record][5]
activeSheet['J' + str(i)] = records[record][6]
record = record + 1
# Date Cell
activeSheet['A5'] = " Bank Al Habib Limited Date:" + \
"01-"+"{:0>2d}".format(int(month))+"-"+year
# Zone Cell
activeSheet['A6'] = "Bills Summary " + zone + " ZONE"
# updating formulas
formulaCells = ['E', 'F', 'G', 'H', 'J']
if noOfRows > 0:
for formulaCell in formulaCells:
activeSheet[formulaCell+str(toRow)] = "=SUM(" + \
formulaCell+str(fromRow)+":"+str(toRow-1)+")"
workbook.save(filename=fileName)
def export_resources(self, part_item):
part_item_file_path = part_item.modified_file_path()
directory = os.path.dirname(part_item_file_path)
filename = os.path.basename(part_item_file_path)
filename_without_extension = os.path.splitext(filename)[0]
#=== subtitles ===
#build up language_code dict
if self.opts.export_subtitles:
logging.warning("attempting to export subtitles...")
subtitle_stream_type = "3"
all_non_embedded_subtitles = []
for stream_item in part_item.stream_items:
if stream_item.stream_type == subtitle_stream_type and stream_item.key != "":
all_non_embedded_subtitles.append(stream_item)
#end if
#end for
number_of_non_embedded_subtitles = len(all_non_embedded_subtitles)
if number_of_non_embedded_subtitles == 0:
logging.warning("no subtitles found")
else:
logging.warning("found %d subtitle(s)" %
number_of_non_embedded_subtitles)
categorized_subtitles = {}
for subtitle in all_non_embedded_subtitles:
key = (subtitle.language_code, subtitle.codec)
if categorized_subtitles.has_key(key):
categorized_subtitles[key].append(subtitle)
else:
categorized_subtitles[key] = [subtitle]
#end if has_key
#end for all_subtitles
exported_subtitles = 0
for key, subtitles in categorized_subtitles.iteritems():
#key = (eng, srt), (eng, sub), (fre, srt), etc.
language_code = key[0]
codec = key[1]
#get all existing sub files. example filename: Sopranos - S01E01 - The Pilot*.eng.srt
compatible_sidecar_subtitles = self.get_all_sidecar_subtitles(
directory, filename_without_extension, language_code,
codec)
if len(compatible_sidecar_subtitles) > 0:
logging.warning(
"Subtitle file(s) with language code '%s' of type '%s' already exist. Skipping all matching..."
% (language_code, codec))
continue
#end if
#export subs
i = 0
for subtitle in subtitles:
if i == 0:
subtitle_filename = "%s.%s.%s" % (
filename_without_extension, language_code,
codec)
else:
subtitle_filename = "%s.%02d.%s.%s" % (
filename_without_extension, i, language_code,
codec)
#end if
subtitle_full_path = os.path.join(
directory, subtitle_filename)
success = subtitle.export_to_path(subtitle_full_path)
if success:
i += 1
exported_subtitles += 1
Summary().subtitle_export_succeeded()
else:
Summary().subtitle_export_failed()
#end success
#end for subtitles
#end for categorized_subtitles
logging.warning(
"exported %d subtitle(s), skipped %d" %
(exported_subtitles,
(number_of_non_embedded_subtitles - exported_subtitles)))
#end if len(all_non_embedded_subtitles) == 0:
#end if subtitles
if self.opts.export_artwork:
logging.error("artwork export not yet implemented...")
def Backtest(self):
'''
temp_df : accept DataFrame with columen:CLOSE HIGH LOW OPEN VOLUME index : datetime object
Stops=10 : int : number of tick to hit stop from entry price
Troubleshoot=False : Boolean : setting to True will also return the source dataframe along with the summary
TP_list=[20,30] : list of length 2 : a pair of targets for from the entry calculated in number of ticks, to set a single target
, put leave the TP_list[1] empty and set TP_size to 1
TP_size=0.5 : float : The proportion of current total position to exit when TP_list[0] reach, set to 1 if exiting all at TP_list[0]
tick_size=0.5: float : the smallest move available in the asset
Slippage=1 : int : the number of ticks to acct of slippage when hitting the Stops
tier_size=1 : int : the number of assset to buy on each signal
CAGR_N=1 : flaot : in number of years, eg 6month ==0.5
'''
self.temp_df['prev_sign'] = self.temp_df['sign'].shift()
self.temp_df['periods'] = np.nan
self.temp_df['EntryP'] = np.nan
self.temp_df['Stop'] = np.nan
self.temp_df['TPs'] = np.nan
EntryPrice, ExitPrice, Position = initialise_setting()
self.temp_df['PL'] = 0
# Loop Thru the Dataframe
for i in range(self.temp_df.shape[0]):
print(f"Row: {i}/{self.temp_df.shape[0]-1}", end='\r')
self.PL = 0
Close = self.temp_df.iloc[
i, self.temp_df.columns.str.upper().get_loc("CLOSE")]
High = self.temp_df.iloc[
i, self.temp_df.columns.str.upper().get_loc("HIGH")]
Low = self.temp_df.iloc[
i, self.temp_df.columns.str.upper().get_loc("LOW")]
OPEN = self.temp_df.iloc[
i, self.temp_df.columns.str.upper().get_loc("OPEN")]
###########################################SL and TP check##########################################
# Assume all position exited with a slippage*tick_size
# as we may not know which direction it will go first, stop out is chosen to be the conservative first direction
############################################## SL ######################################
# Stop_hit(ExitPrice,EntryPrice,Low,High,Position)
# - if Stops hit
# - LowExitPrice,EntryPrice,,High,Position will be re-initialized
# - self.PL = Position * (ExitPrice['Stop'] - EntryPrice) - self.Slippage
# Else it will return the input value for ExitPrice,EntryPrice,Low,High,Position
EntryPrice, ExitPrice, Position = self.Stop_Hit(
ExitPrice, EntryPrice, Low, High, Position)
################################################ TP ###############################################
# TP_hit(ExitPrice,EntryPrice,Low,High,Position)
# - if Stops hit
# - LowExitPrice,EntryPrice,,High,Position will be re-initialized
# - self.PL = Position * (ExitPrice['Stop'] - EntryPrice) - self.Slippage
# Else it will return the input value for ExitPrice,EntryPrice,Low,High,Position
EntryPrice, ExitPrice, Position = self.TP_hit(
ExitPrice, EntryPrice, Low, High, Position)
EntryPrice, ExitPrice, Position = self.New_position(
i, ExitPrice, EntryPrice, Position, Close)
# CLOSE ALL at last period
if i == self.temp_df.shape[0] - 1:
if Position != 0:
self.PL = Position * (Close - EntryPrice)
# Append the PL for this Epoch to the List later to become the PL column
self.PL_list.append(self.PL)
######################## Outside for loop calculate summaries ###############################################
self.temp_df['PL'] = self.PL_list
self.temp_df['Cumm_PL'] = self.temp_df['PL'].cumsum()
# adding to summary data to a Dataframe
Summary_dict = S.summary(self.temp_df.PL,
ref_price=self.temp_df['OPEN'].iloc[0],
n=self.CAGR_N).Summary_dict
Summary_dict['Period'] = 'Start: ' + str(
self.temp_df.index[0]) + ' End: ' + str(self.temp_df.index[-1])
Summary_dict['NO. of Buy signal'] = self.temp_df['Actions'][
self.temp_df['Actions'] == 'BUY'].count()
Summary_dict['NO. of Sell signal'] = self.temp_df['Actions'][
self.temp_df['Actions'] == 'SELL'].count()
#print(Summary_dict)
if self.Troubleshoot == True or self.Return_df == True:
return [Summary_dict, self.temp_df]
return Summary_dict
def run(self):
"""Thread entry point"""
sRunError = ""
self.userSourceIDs = True
log.debug("Running Sequence")
try:
# Main sequence run loop
log.debug("sourceSequence -> %s" % str(self.sourceSequence))
sMessage = "Run of sequence \"" + self.sourceSequence["metadata"]["name"] + "\" started."
log.debug(sMessage)
self.messaging.broadcastMessage(sMessage)
nComponentCount = len(self.sourceSequence["components"])
nCurrentComponent = 0
while nCurrentComponent < nComponentCount:
# Get the next component
pSourceComponent = self.sourceSequence["components"][nCurrentComponent]
# Skip components until we find our start component
self.sourceComponentID = pSourceComponent["id"]
if self.initializing and (self.startComponentID != 0) and (self.sourceComponentID != self.startComponentID):
log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")")
nCurrentComponent += 1
continue
# Update our initializing and running flags
self.initializing = False
self.running = True
# Ignore any previous summary component
if pSourceComponent["componenttype"] == Summary.componentType:
log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")")
nCurrentComponent += 1
continue
# Create and run the next unit operation
log.debug("Starting unit operation " + str(self.sourceComponentID) + " (" +
pSourceComponent["componenttype"] + ")")
pSourceUnitOperation = UnitOperations.createFromComponent(self.sourceSequenceID, pSourceComponent, self.username, self.sequenceManager.database, self.systemModel)
self.runComponentID = pSourceUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID)
pRunComponent = self.sequenceManager.GetComponent(self.username, self.runComponentID, self.runSequenceID)
pRunUnitOperation = UnitOperations.createFromComponent(self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel)
pRunUnitOperation.setDaemon(True)
pRunUnitOperation.start()
self.systemModel.SetUnitOperation(pRunUnitOperation)
log.debug( "Unit Op is alive")
# Wait until the operation completes or we receive an abort signal
while pRunUnitOperation.is_alive() and not self.runAborted:
log.debug( "Unit Op check alive")
time.sleep(0.25)
self.systemModel.SetUnitOperation(None)
if self.runAborted:
log.error( "Unit Op ABORTED!")
pRunUnitOperation.setAbort()
raise Exception("Run aborted")
# Check for unit operation error
log.debug( "Unit check for error")
sError = pRunUnitOperation.getError()
if sError != "":
log.error("UnitOperation: %s" % pRunUnitOperation.__class__.__name__)
log.error( "Unit operation failed: " + sError)
raise Exception(sError)
log.debug("Prepare to update operation details")
# Update the unit operation details in the database
UnitOperations.updateToComponent(pRunUnitOperation, self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel)
log.debug("After updateToComponent")
self.sequenceManager.UpdateComponent(self.username, self.runSequenceID, self.runComponentID, None, pRunComponent)
log.debug("After updateComponent")
log.debug("Completed unit operation (" + pRunComponent["componenttype"] + ")")
self.sourceComponentID = 0
self.runComponentID = 0
# Check if the user paused the sequence for editing
if self.runWillPause:
# Pause until editing is complete
log.debug("Pausing run for editing")
self.runWillPause = False
self.runIsPaused = True
while self.runIsPaused and not self.runAborted:
time.sleep(0.25)
if self.runAborted:
raise Exception("Sequence aborted")
log.debug("Continuing paused run")
# Reload the sequence and make sure it is still valid
self.sourceSequence = self.sequenceManager.GetSequence(self.username, self.sourceSequenceID)
if not self.sourceSequence["metadata"]["valid"]:
raise Exceptions("Edited sequence is no longer valid (" + self.sourceSequenceID + ")")
# Scan until we find the unit operation we just executed
nComponentCount = len(self.sourceSequence["components"])
nCurrentComponent = 0
while nCurrentComponent < nComponentCount:
if self.sourceSequence["components"][nCurrentComponent]["id"] == pSourceComponent["id"]:
break
nCurrentComponent += 1
if nCurrentComponent == nComponentCount:
raise Exception("Failed to find previous component in edited sequence")
# Advance to the next component
nCurrentComponent += 1
except Exception as ex:
log.error("Sequence run failed: " + str(ex))
log.error("Trace Back %s" % traceback.format_exc())
sRunError = str(ex)
# Add the Summary unit operation to the sequence
pSummaryComponent = Summary.createNewComponent(sRunError)
self.runComponentID = self.database.CreateComponent(self.username, self.runSequenceID, pSummaryComponent["type"], pSummaryComponent["note"], json.dumps(pSummaryComponent))
# Fully validate the run sequence
self.sequenceManager.ValidateSequenceFull(self.username, self.runSequenceID)
# Switch to using the run IDs rather than the source because the summary unit operation only exists in the run sequence
self.userSourceIDs = False
# Instantiate and start the summary unit operation
log.info("Starting summary unit operation")
pSummaryComponent = self.sequenceManager.GetComponent(self.username, self.runComponentID, self.runSequenceID)
pSummaryUnitOperation = UnitOperations.createFromComponent(self.runSequenceID, pSummaryComponent, self.username, self.sequenceManager.database, self.systemModel)
pSummaryUnitOperation.setDaemon(True)
pSummaryUnitOperation.start()
self.systemModel.SetUnitOperation(pSummaryUnitOperation)
# Broadcast the summary unit operation message
self.messaging.broadcastMessage(pSummaryComponent["message"])
# Wait until the operation completes
while pSummaryUnitOperation.is_alive():
time.sleep(0.25)
log.info("Summary unit operation complete")
self.runComponentID = 0
# Run complete
log.debug("Run stopped")
self.running = False
def upload_file():
try:
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
uploaded_files = request.files.getlist("file")
milliseconds = request.args.get("milliseconds")
rules = request.args.get("rules")
rules = rules.split(',')
link = 'https://s3.us-east.cloud-object-storage.appdomain.cloud/sharad-saurav-bucket/DataFiles_Rules_Report.xlsx'
target, headers = urllib.request.urlretrieve(link)
print(target, headers)
newTar = target + ".xlsx"
os.rename(target, newTar)
length = len(rules)
numberOfFiles = len(uploaded_files)
fileNames = []
for file in uploaded_files:
fileNames.append(file.filename)
for file in uploaded_files:
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
Check_Columns.checkColumn(file, filename)
if ("Allowed_intents_in_Unstructured" in rules):
Allowed_intents_in_Unstructured.rule_unstructured(
file, filename, newTar)
if ("Check_for_duplicates" in rules):
Check_for_duplicates.rule_duplicates(
file, filename, newTar)
if ("Check_for_missing_Keyword" in rules):
Check_for_missing_Keyword.rule_missing_keyword(
file, filename, newTar)
if ("Check_for_capitalization" in rules):
Check_for_capitalization.rule_capitalization(
file, filename, newTar)
if ("Check_if_date_time_are_blank" in rules):
Check_if_date_time_are_blank.rule_date_time_blank(
file, filename, newTar)
if ("Correctness_of_MAP_URL" in rules):
Correctness_of_MAP_URL.rule_map_url(file, filename, newTar)
if ("Correctness_of_Short_Ref_URL" in rules):
Correctness_of_Short_Ref_URL.short_ref_url(
file, filename, newTar)
if ("Date_in_YYYY_MM_DD_format" in rules):
Date_in_YYYY_MM_DD_format.date_format(
file, filename, newTar)
if ("Description_text_not_same" in rules):
Description_text_not_same.description_text(
file, filename, newTar)
if ("Duplicate_in_Entity_Interactn" in rules):
Duplicate_in_Entity_Interactn.duplicate_entity_interaction(
file, filename, newTar)
if ("Email_id_validity" in rules):
Email_id_validity.email_id_validity(file, filename, newTar)
if ("Exact_dates_available" in rules):
Exact_dates_available.exact_dates_available(
file, filename, newTar)
if ("Exceeding_500_characters" in rules):
Exceeding_500_characters.exceeding_500_characters(
file, filename, newTar)
if ("Latitude_Longitude" in rules):
Latitude_Longitude.latitide_longitude(
file, filename, newTar)
if ("Multiple_Spaces_in_txt" in rules):
Multiple_Spaces_in_txt.multiple_spaces_in_txt(
file, filename, newTar)
if ("No_content_in_brackets" in rules):
No_content_in_brackets.no_content_in_brackets(
file, filename, newTar)
if ("No_AcadEvents_in_Timing" in rules):
No_AcadEvents_in_Timing.no_acadEvents_in_timing(
file, filename, newTar)
if ("No_date_special_characters" in rules):
No_date_special_characters.No_date_special_characters(
file, filename, newTar)
if ("No_phone_url_in_voice" in rules):
No_phone_url_in_voice.no_phone_url_in_voice(
file, filename, newTar)
if ("No_preceeding_0_in_room_no" in rules):
No_preceeding_0_in_room_no.no_preceeding_0_in_room_no(
file, filename, newTar)
if ("No_Ref_URL_in_text" in rules):
No_Ref_URL_in_text.no_ref_url_in_text(
file, filename, newTar)
if ("No_sentence_in_voice_column" in rules):
# No_sentence_in_virtual_entity.no_sentence_in_virtual_entity(file, filename, newTar)
No_sentence_in_voice_column.no_sentence_in_voice_column(
file, filename, newTar)
if ("No_timing_for_acad_events" in rules):
No_timing_for_acad_events.no_timing_for_acad_events(
file, filename, newTar)
if ("No_timings_values_in_txt" in rules):
No_timings_values_in_txt.no_timings_values_in_txt(
file, filename, newTar)
if ("Numbering_bullet_points" in rules):
Numbering_bullet_points.numbering_bullet_points(
file, filename, newTar)
if ("Process_ID" in rules):
# Perfect_Excel_format.perfect_excel_format(file, filename, newTar)
Process_ID.process_id(file, filename, newTar)
if ("Special_Char_in_Entity_Name" in rules):
Special_Char_in_Entity_Name.special_char_in_entity_name(
file, filename, newTar)
if ("Start_date_less_than_end_date" in rules):
Start_date_less_than_end_date.start_date_less_than_end_date(
file, filename, newTar)
if ("Start_time_less_than_end_time" in rules):
Start_time_less_than_end_time.start_time_less_than_end_time(
file, filename, newTar)
if ("Time_in_HH_MM_SS_format" in rules):
Time_in_HH_MM_SS_format.time_in_hh_mm_ss_format(
file, filename, newTar)
Summary.summary(newTar, numberOfFiles, rules, fileNames)
uploadFile.multi_part_upload(
"sharad-saurav-bucket",
"DataFiles_Rules_Report" + milliseconds + ".xlsx", newTar)
return getJson.get_Json_data(newTar, length)
except Exception as e:
traceback.print_exc()
return str(e)
