def remove_tags(self, part_item): SublerCLI = os.path.join(sys.path[0], "SublerCLI-v010") filepath = part_item.modified_file_path() #removal of artwork doesn't seem to work all_tags = ["{Artwork:}", "{HD Video:}", "{Gapless:}", "{Content Rating:}", "{Media Kind:}", "{Name:}", "{Artist:}", "{Album Artist:}", "{Album:}", "{Grouping:}", "{Composer:}", "{Comments:}", "{Genre:}", "{Release Date:}", "{Track #:}", "{Disk #:}", "{TV Show:}", "{TV Episode #:}", "{TV Network:}", "{TV Episode ID:}", "{TV Season:}", "{Description:}", "{Long Description:}", "{Rating:}", "{Rating Annotation:}", "{Studio:}", "{Cast:}", "{Director:}", "{Codirector:}", "{Producers:}", "{Screenwriters:}", "{Lyrics:}", "{Copyright:}", "{Encoding Tool:}", "{Encoded By:}", "{contentID:}"]#these are currently not supported in subler cli tool, "{XID:}", "{iTunes Account:}", "{Sort Name:}", "{Sort Artist:}", "{Sort Album Artist:}", "{Sort Album:}", "{Sort Composer:}", "{Sort TV Show:}"] logging.warning("removing tags...") #Create the command line command tag_removal_cmd = ['%s' % SublerCLI] if self.opts.optimize: action_description = "Tags removed and optimized" tag_removal_cmd.append("-O") else: action_description = "Tags removed" #end if optimize tag_removal_cmd.append("-t") tag_removal_cmd.append("".join(all_tags)) tag_removal_cmd.append("-i") tag_removal_cmd.append(filepath) success = self.execute_command(filepath, tag_removal_cmd, action_description) if success: Summary().metadata_removal_succeeded() else: Summary().metadata_removal_failed()
def tag(self, part_item): SublerCLI = os.path.join(sys.path[0], "SublerCLI") filepath = part_item.modified_file_path() directory = os.path.dirname(filepath) filename = os.path.basename(filepath) filename_without_extension = os.path.splitext(filename)[0] logging.warning("tagging...") #Create the command line command tag_cmd = ['%s' % SublerCLI] if self.opts.optimize: action_description = "Tags added and optimized" tag_cmd.append("-optimize") else: action_description = "Tags added" #end if optimize if self.opts.embed_subtitles: action_description += ", and any subtitles embedded" compatible_sidecar_subtitles = self.get_all_sidecar_subtitles( directory, filename_without_extension, codec='srt') if len(compatible_sidecar_subtitles) == 0: logging.warning("Found no 'srt' subtitle files to embed...") else: for sub in compatible_sidecar_subtitles: tag_cmd.append("-s") tag_cmd.append(os.path.join(directory, sub)) if self.opts.chapter_previews: action_description += ", and chapter previews generated" tag_cmd.append("-p") tag_cmd.append("-metadata") tag_cmd.append(part_item.tag_string()) #also downloads the artwork tag_cmd.append("-source") tag_cmd.append(filepath) tag_cmd.append("-destination") tmp_filepath = filepath + "_tmp" tag_cmd.append(tmp_filepath) success = self.execute_command(filepath, tag_cmd, action_description) if success: Summary().metadata_embedded_succeeded() logging.info("Replacing old file with tagged one...") shutil.move(tmp_filepath, filepath) else: Summary().metadata_embedded_failed() logging.info("Cleaning up temporary file...") os.unlink(tmp_filepath)
def update_model(): if memory.full: replay = memory.sample(batch_size) state_list = [] target_list = [] for sample in replay: input = sample[0] action = sample[1] reward_list = sample[2] possible_states = sample[3] targetQ = [] _, allQ = sess.run([predict, Qout], feed_dict={inputs: [input]}) if update_mode == "single": next_state = possible_states[action] next_input = normalize(next_state.grid_to_input()) Q1 = sess.run(Qout, feed_dict={inputs: [next_input]}) maxQ1 = np.max(Q1) targetQ = allQ targetQ[0, action] = reward_list[action] + \ (0 if next_state.halt else gamma * maxQ1) elif update_mode == "all": next_inputs = [ normalize(s.grid_to_input()) for s in possible_states ] Q1 = sess.run(Qout, feed_dict={inputs: next_inputs}) maxQs = [np.max(Q) for Q in Q1] targetQ = allQ for k in range(4): if possible_states[k].valid: targetQ[0, k] = reward_list[k] + \ (0 if possible_states[k].halt else gamma * maxQs[k]) state_list.insert(0, input) target_list.insert(0, targetQ[0]) _, summary = sess.run([updateModel, summary_op], feed_dict={ inputs: state_list, nextQ: target_list }) Summary.write_summary_operation(summary, total_steps + steps)
def updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel = None): if pComponent["componenttype"] == Cassette.componentType: return Cassette.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Add.componentType: return Add.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Evaporate.componentType: return Evaporate.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Transfer.componentType: return Transfer.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == React.componentType: return React.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Prompt.componentType: return Prompt.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Install.componentType: return Install.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Comment.componentType: return Comment.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == TrapF18.componentType: return TrapF18.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == EluteF18.componentType: return EluteF18.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Initialize.componentType: return Initialize.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Mix.componentType: return Mix.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Move.componentType: return Move.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Summary.componentType: return Summary.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == ExternalAdd.componentType: return ExternalAdd.updateToComponent(pUnitOperation, nSequenceID, pComponent, username, database, systemModel) else: raise Exception("Unknown component type: " + pComponent["componenttype"])
def add_to_itunes(self, part_item): if self.video_item.__class__.__name__ == "MovieItem": itunes_playlist = "Movies" else: itunes_playlist = "TV Shows" item_title = self.video_item.title actionable_file_path = part_item.modified_file_path() if not self.opts.force: logging.warning("Finding '%s' in iTunes..." % actionable_file_path) search_string = 'set currentItems to search playlist "%s" for "%s" only displayed' % (itunes_playlist, item_title) delimiter = "###" result_creation_string = 'set output to output & "%s"' % delimiter does_item_exist_command = ["osascript", '-e', 'tell application "iTunes"', '-e', 'try' ,'-e', search_string, '-e', 'set output to ""', '-e', 'repeat with currentItem in currentItems', '-e', 'set loc to (location of currentItem)', '-e', 'if output is not equal to "" then', '-e', result_creation_string, '-e', 'end if', '-e', 'set output to output & POSIX path of loc', '-e', 'end repeat', '-e', 'output', '-e', 'end try', '-e', 'end tell'] logging.debug("'find in iTunes' script: %s" % (does_item_exist_command)) result_string = subprocess.Popen(does_item_exist_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn = self.preexec).communicate()[0].rstrip() result_file_paths = result_string.split(delimiter) logging.debug("Find results in iTunes: %s" % result_file_paths) for file_path in result_file_paths: unicode_filepath = normalize('NFC', file_path.decode('utf-8')) if unicode_filepath == actionable_file_path: logging.warning(" Already added to iTunes") return #end if result == actionable_file_path: #end for file_path in result_file_paths #end if not self.opts.force logging.warning(" Adding to iTunes...") file_str = 'set p to (POSIX file "%s")' % actionable_file_path add_to_playlist_str = 'add p to playlist "%s"' % itunes_playlist add_to_itunes_command = ['osascript', '-e', 'try', '-e', file_str, '-e', 'tell application "iTunes"', '-e', add_to_playlist_str, '-e', 'end tell', '-e', 'end try'] logging.debug("'add to iTunes' script: %s" % (add_to_itunes_command)) result = subprocess.Popen(add_to_itunes_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn = self.preexec).communicate()[0].rstrip() if result.startswith("file track id"): Summary().add_to_itunes_succeeded() logging.warning(" Adding additional metadata to item in iTunes...") current_item_str = 'tell %s' % result set_rating_str = 'set rating to %i as integer' % self.video_item.itunes_rating() set_play_count_str = 'set played count to %s as integer' % self.video_item.view_count add_metadata_to_itunes_command = ['osascript', '-e', 'try', '-e', 'tell application "iTunes"', '-e', current_item_str, '-e', set_play_count_str, '-e', set_rating_str, '-e', 'end tell', '-e', 'end tell', '-e', 'end try'] logging.debug("'add additional metadata iTunes' script: %s" % (add_metadata_to_itunes_command)) subprocess.call(add_metadata_to_itunes_command) else: Summary().add_to_itunes_failed() logging.critical("Failed 'add to iTunes' for '%s'. Incorrect path or not a compatible file type?" % (actionable_file_path) )
def optimize(self, part_item): SublerCLI = os.path.join(sys.path[0], "SublerCLI-v010") filepath = part_item.modified_file_path() logging.warning("optimizing file...") action_description = "Tags optimized" #Create the command line command optimize_cmd = ['%s' % SublerCLI] optimize_cmd.append("-O") optimize_cmd.append("-i") optimize_cmd.append(filepath) success = self.execute_command(filepath, optimize_cmd, action_description) if success: Summary().metadata_optimized_succeeded() else: Summary().metadata_optimized_failed()
def process(self): skipped_all = True if self.opts.gather_statistics: skipped_all = False LibraryStatistics().add_item(self.video_item) #end if gather_statistics for index, media_item in enumerate(self.media_items): logging.warning("processing %d/%d media_items" % (index + 1, len(self.media_items))) part_items = media_item.part_items for index2, part_item in enumerate(part_items): no_action = True logging.warning(" processing %d/%d part_items" % (index2 + 1, len(part_items))) Summary().increment_parts_processed() if self.opts.removetags and self.canTag(part_item): skipped_all = no_action = False self.remove_tags(part_item) #end if removetags if self.opts.tag and self.canTag(part_item) and self.shouldTag( part_item): skipped_all = no_action = False self.tag(part_item) #end if tag if self.opts.optimize and not self.opts.tag and not self.opts.removetags and self.canTag( part_item): #optimize is done together with tagging or removing, so only needs to be done here if it's the exclusive action skipped_all = no_action = False self.optimize(part_item) #end if optimize if self.opts.export_resources: skipped_all = no_action = False self.export_resources(part_item) #end if export_resouces if self.opts.add_to_itunes: skipped_all = no_action = False self.add_to_itunes(part_item) #end if add_to_itunes if self.opts.open_file_location and not no_action: logging.warning("opening '%s'..." % part_item.modified_file_path()) command = ['open', "-R", part_item.modified_file_path()] subprocess.call(command) #end if self.opts.open_file_location #end for part_items #end for media_items if skipped_all: logging.warning( "skipping: found no suitable files for specified tasks") #end if skipped_all #end def process_item #end MediaPartProcessor
def createFromComponent(nSequenceID, pComponent, username, database, systemModel=None): if pComponent["componenttype"] == Cassette.componentType: return Cassette.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Add.componentType: return Add.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Evaporate.componentType: return Evaporate.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Transfer.componentType: return Transfer.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == React.componentType: return React.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Prompt.componentType: return Prompt.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Install.componentType: return Install.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Comment.componentType: return Comment.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == TrapF18.componentType: return TrapF18.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == EluteF18.componentType: return EluteF18.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Initialize.componentType: return Initialize.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Mix.componentType: return Mix.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Move.componentType: return Move.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == Summary.componentType: return Summary.createFromComponent(nSequenceID, pComponent, username, database, systemModel) elif pComponent["componenttype"] == ExternalAdd.componentType: return ExternalAdd.createFromComponent(nSequenceID, pComponent, username, database, systemModel) else: raise Exception("Unknown component type: " + pComponent["componenttype"])
def __init__(self, y, x_global, x_local, kernel, mType=0, offset=None, sigma2_v1=False, tol=1.0e-6, maxIter=200, y_name="", y_off_name="", x_name_glob=[], x_name_loc=[], fle_name="", summaryGLM=False): """ Initialize class """ semiGWR_Base.__init__(self, y, x_global, x_local, kernel, mType, offset, sigma2_v1, tol, maxIter) self.y_name = y_name self.x_name_glob = x_name_glob self.x_name_loc = x_name_loc if y_off_name is None: y_off_name = " " self.y_off_name = y_off_name # add x names-global n_xname = len(self.x_name_glob) if n_xname < self.nVars_glob: for i in range(n_xname,self.nVars_glob): self.x_name_glob.append("name" + str(i)) # add x names-local n_xname = len(self.x_name_loc) if n_xname < self.nVars_loc: for i in range(n_xname,self.nVars_loc-1): self.x_name_loc.append("name" + str(i)) if len(self.x_name_loc) < self.nVars_loc: self.x_name_loc.insert(0,'Intercept') self.fle_name = fle_name self.mName = semiGWR_Names[self.mType] if summaryGLM: x = np.hstack((x_global,x_local)) x_name = [ elem for elem in x_name_glob] x_name.extend(x_name_loc) #if mType == 0: #self.OLS = OLS(y,x,sigma2_v1,y_name,x_name,fle_name) #else: self.GLM = GLM(y, x, mType, offset, sigma2_v1, tol, maxIter, y_name, y_off_name, x_name, fle_name) Summary.semiGWR(self)
def __init__(self, y, x, kernel, mType=0, offset=None, sigma2_v1=False, tol=1.0e-6, maxIter=200, y_name="", y_off_name="", x_name=[], fle_name="", summaryGLM=False): """ Initialize class """ GWGLM_Base.__init__(self, y, x, kernel, mType, offset, None, None, None, sigma2_v1, tol, maxIter) self.y_name = y_name self.x_name = x_name if y_off_name is None: y_off_name = " " self.y_off_name = y_off_name # add x names n_xname = len(self.x_name) if n_xname < self.nVars: for i in range(n_xname, self.nVars - 1): self.x_name.append("name" + str(i)) n_xname = len(self.x_name) if n_xname < self.nVars: self.x_name.insert(0, 'Intercept') self.fle_name = fle_name self.mName = gwglm_Names[self.mType] if summaryGLM: #if mType == 0: #self.OLS = OLS(y,x,sigma2_v1,y_name,x_name,fle_name) #else: self.GLM = GLM(y, x, mType, offset, sigma2_v1, tol, maxIter, y_name, y_off_name, x_name, fle_name) #if mType == 0: #Summary.GWR(GWRMod=self) #else: Summary.GWGLM(self)
def __init__(self, parent): wx.Notebook.__init__(self, parent, id=wx.ID_ANY, style=wx.BK_DEFAULT) self.AddPage(CourtCase.CourtCase(self), "Court Case") self.AddPage(Reference.NestedReferencePanel(self), "Reference") self.AddPage(Official.OfficialPanel(self), "Official") self.AddPage(Defendant.NestedDefendantPanel(self), "Defendant") self.AddPage(Plaintiff.NestedPlaintiffPanel(self), "Plaintiff") self.AddPage(Prosecutor.NestedProsecutorPanel(self), "Prosecutor") self.AddPage(Witness.NestedWitnessPanel(self), "Witness") self.AddPage(Charges.ChargesPanel(self), "Charges") self.AddPage(Summary.SummaryPanel(self), "Summary") self.AddPage(CaseNotes.CaseNotesPanel(self), "Case Notes") self.AddPage(HealingNotes.HealingNotesPanel(self), "Healing Notes") self.AddPage(FurtherNotes.FurtherNotesPanel(self), "Further Research Notes")
def MiniBatchTrain(config, load_model=False): # Set learning parameters gamma = config["discount-factor"] num_episodes = config["epochs"] learning_rate = config["learning-rate"] [eps_start, eps_stop, eps_steps] = config["epsilon-params"] model_id = config["model-id"] memory_size = config["replay-size"] exploration = Exploration.getExplorationFromArgs(config["exploration"]) batch_size = config["batch-size"] update_mode = config["update-mode"] tensorboard_port = config["tensorboard"] # Initialize TF Network and variables Qout, inputs = Network.getNetworkFromArgs(config["architecture"]) Qmean = tf.reduce_mean(Qout) Qmax = tf.reduce_max(Qout) predict = tf.argmax(Qout, 1) # Initialize TF output and optimizer nextQ = tf.placeholder(shape=[None, 4], dtype=tf.float32) loss = Losses.getLossFromArgs(config["loss"])(nextQ, Qout) trainer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate) updateModel = trainer.minimize(loss) init = tf.global_variables_initializer() # Initialize tensorboard summary summary_op = Summary.init_summary_writer(model_name=model_id, var_list=[("loss", loss), ("Qmean", Qmean), ("Qmax", Qmax)], tb_port=tensorboard_port) # Random action parameter _epsilon = Gradients.Exponential(start=eps_start, stop=eps_stop) def epsilon(step): # Exponentially decreasing epsilon to 0.1 for first 25% epochs, constant value of 0.1 from there on if step < num_episodes / (100.0 / eps_steps): return _epsilon(step / (num_episodes / (100.0 / eps_steps))) else: return eps_stop memory = ReplayMemory(memory_size) def update_model(): if memory.full: replay = memory.sample(batch_size) state_list = [] target_list = [] for sample in replay: input = sample[0] action = sample[1] reward_list = sample[2] possible_states = sample[3] targetQ = [] _, allQ = sess.run([predict, Qout], feed_dict={inputs: [input]}) if update_mode == "single": next_state = possible_states[action] next_input = normalize(next_state.grid_to_input()) Q1 = sess.run(Qout, feed_dict={inputs: [next_input]}) maxQ1 = np.max(Q1) targetQ = allQ targetQ[0, action] = reward_list[action] + \ (0 if next_state.halt else gamma * maxQ1) elif update_mode == "all": next_inputs = [ normalize(s.grid_to_input()) for s in possible_states ] Q1 = sess.run(Qout, feed_dict={inputs: next_inputs}) maxQs = [np.max(Q) for Q in Q1] targetQ = allQ for k in range(4): if possible_states[k].valid: targetQ[0, k] = reward_list[k] + \ (0 if possible_states[k].halt else gamma * maxQs[k]) state_list.insert(0, input) target_list.insert(0, targetQ[0]) _, summary = sess.run([updateModel, summary_op], feed_dict={ inputs: state_list, nextQ: target_list }) Summary.write_summary_operation(summary, total_steps + steps) with tf.Session() as sess: sess.run(init) total_steps = 0 for i in range(num_episodes): # Reset environment and get first new observation state = Game.new_game(4) reward_sum = 0 steps = 0 rand_steps = 0 invalid_steps = 0 # The Q-Network while not state.halt: s = normalize(state.grid_to_input()) steps += 1 if i == 0: state.printstate() print "" # Choose an action by greedily (with e chance of random action) from the Q-network a, allQ = sess.run([predict, Qout], feed_dict={inputs: [s]}) possible_states, action, ra, invalid_prediction = exploration( a[0], allQ, i, epsilon, state) if ra: rand_steps += 1 if invalid_prediction: invalid_steps += 1 reward_list = [] for k, nextstate in enumerate(possible_states): r = reward(state, nextstate) if r is not 0: r = np.log2(nextstate.score - state.score) / 2.0 reward_list.insert(k, r) reward_sum += reward_list[action] # [normailzed input, action_taken, rewards for all action, all possible states] memory.push([s, action, reward_list, possible_states]) # update step update_model() state = possible_states[action] maxtile = max([max(state.grid[k]) for k in range(len(state.grid))]) stat = { 'max-tile': maxtile, 'score': state.score, 'steps': steps, 'r': reward_sum, 'rand-steps': "{0:.3f}".format(float(rand_steps) / steps) } total_steps += steps Summary.write_scalar_summaries( [ ("steps", steps), ("epsilon", epsilon(i)), ("score", state.score), ("rand-steps", float(rand_steps) / steps), ("maxtile", maxtile), # ("invalid-steps", steps) ], i) print i, "\t", stat sess.close()
import Genetic as g import Summary as s sentences = [s.Simple_Sentence("hello there"), s.Simple_Sentence("am"), s.Simple_Sentence("A big boy now")] target = [1, 0, 2] pop = g.population(5) for i in pop: i.calc_rankings(sentences) print i.weights print i.rankings print g.fitness(i, target) print "----"
def run(self): """Thread entry point""" sRunError = "" self.userSourceIDs = True log.debug("Running Sequence") try: # Main sequence run loop log.debug("sourceSequence -> %s" % str(self.sourceSequence)) sMessage = "Run of sequence \"" + self.sourceSequence["metadata"][ "name"] + "\" started." log.debug(sMessage) self.messaging.broadcastMessage(sMessage) nComponentCount = len(self.sourceSequence["components"]) nCurrentComponent = 0 while nCurrentComponent < nComponentCount: # Get the next component pSourceComponent = self.sourceSequence["components"][ nCurrentComponent] # Skip components until we find our start component self.sourceComponentID = pSourceComponent["id"] if self.initializing and (self.startComponentID != 0) and ( self.sourceComponentID != self.startComponentID): log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")") nCurrentComponent += 1 continue # Update our initializing and running flags self.initializing = False self.running = True # Ignore any previous summary component if pSourceComponent["componenttype"] == Summary.componentType: log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")") nCurrentComponent += 1 continue # Create and run the next unit operation log.debug("Starting unit operation " + str(self.sourceComponentID) + " (" + pSourceComponent["componenttype"] + ")") pSourceUnitOperation = UnitOperations.createFromComponent( self.sourceSequenceID, pSourceComponent, self.username, self.sequenceManager.database, self.systemModel) self.runComponentID = pSourceUnitOperation.copyComponent( self.sourceSequenceID, self.runSequenceID) pRunComponent = self.sequenceManager.GetComponent( self.username, self.runComponentID, self.runSequenceID) pRunUnitOperation = UnitOperations.createFromComponent( self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel) pRunUnitOperation.setDaemon(True) pRunUnitOperation.start() self.systemModel.SetUnitOperation(pRunUnitOperation) log.debug("Unit Op is alive") # Wait until the operation completes or we receive an abort signal while pRunUnitOperation.is_alive() and not self.runAborted: log.debug("Unit Op check alive") time.sleep(0.25) self.systemModel.SetUnitOperation(None) if self.runAborted: log.error("Unit Op ABORTED!") pRunUnitOperation.setAbort() raise Exception("Run aborted") # Check for unit operation error log.debug("Unit check for error") sError = pRunUnitOperation.getError() if sError != "": log.error("UnitOperation: %s" % pRunUnitOperation.__class__.__name__) log.error("Unit operation failed: " + sError) raise Exception(sError) log.debug("Prepare to update operation details") # Update the unit operation details in the database UnitOperations.updateToComponent(pRunUnitOperation, self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel) log.debug("After updateToComponent") self.sequenceManager.UpdateComponent(self.username, self.runSequenceID, self.runComponentID, None, pRunComponent) log.debug("After updateComponent") log.debug("Completed unit operation (" + pRunComponent["componenttype"] + ")") self.sourceComponentID = 0 self.runComponentID = 0 # Check if the user paused the sequence for editing if self.runWillPause: # Pause until editing is complete log.debug("Pausing run for editing") self.runWillPause = False self.runIsPaused = True while self.runIsPaused and not self.runAborted: time.sleep(0.25) if self.runAborted: raise Exception("Sequence aborted") log.debug("Continuing paused run") # Reload the sequence and make sure it is still valid self.sourceSequence = self.sequenceManager.GetSequence( self.username, self.sourceSequenceID) if not self.sourceSequence["metadata"]["valid"]: raise Exceptions( "Edited sequence is no longer valid (" + self.sourceSequenceID + ")") # Scan until we find the unit operation we just executed nComponentCount = len(self.sourceSequence["components"]) nCurrentComponent = 0 while nCurrentComponent < nComponentCount: if self.sourceSequence["components"][ nCurrentComponent]["id"] == pSourceComponent[ "id"]: break nCurrentComponent += 1 if nCurrentComponent == nComponentCount: raise Exception( "Failed to find previous component in edited sequence" ) # Advance to the next component nCurrentComponent += 1 except Exception as ex: log.error("Sequence run failed: " + str(ex)) log.error("Trace Back %s" % traceback.format_exc()) sRunError = str(ex) # Add the Summary unit operation to the sequence pSummaryComponent = Summary.createNewComponent(sRunError) self.runComponentID = self.database.CreateComponent( self.username, self.runSequenceID, pSummaryComponent["type"], pSummaryComponent["note"], json.dumps(pSummaryComponent)) # Fully validate the run sequence self.sequenceManager.ValidateSequenceFull(self.username, self.runSequenceID) # Switch to using the run IDs rather than the source because the summary unit operation only exists in the run sequence self.userSourceIDs = False # Instantiate and start the summary unit operation log.info("Starting summary unit operation") pSummaryComponent = self.sequenceManager.GetComponent( self.username, self.runComponentID, self.runSequenceID) pSummaryUnitOperation = UnitOperations.createFromComponent( self.runSequenceID, pSummaryComponent, self.username, self.sequenceManager.database, self.systemModel) pSummaryUnitOperation.setDaemon(True) pSummaryUnitOperation.start() self.systemModel.SetUnitOperation(pSummaryUnitOperation) # Broadcast the summary unit operation message self.messaging.broadcastMessage(pSummaryComponent["message"]) # Wait until the operation completes while pSummaryUnitOperation.is_alive(): time.sleep(0.25) log.info("Summary unit operation complete") self.runComponentID = 0 # Run complete log.debug("Run stopped") self.running = False
model_path = './model' tf.reset_default_graph() """ Creates the master worker that maintains the master network. We then initialize the workers array. """ global_episodes = tf.Variable(0, dtype=tf.int32, name='global_episodes', trainable=False) total_frames = tf.Variable(0, dtype=tf.int32, name='total_frames', trainable=False) learning_rate = tf.train.polynomial_decay(LEARNING_RATE, total_frames, MAX_ITERATION // 2, LEARNING_RATE * 0.1) with tf.device("/cpu:0"): summary_writer = tf.summary.FileWriter("./train/" + SUMMARY_NAME + str(BETA[BETA_i])) summary = Summary(summary_writer) master_worker = Worker('global', env, GAMMA, learning_rate, global_episodes, total_frames, model_path, False, False, num_workers, summary, BETA) workers = [] for i in range(num_workers): print (i) workers.append( Worker(i, env, GAMMA, learning_rate, global_episodes, total_frames, model_path, render, save_img, num_workers, summary, BETA[BETA_i])) """ Initializes tensorflow variables """ with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=1)) as session: saver = tf.train.Saver(max_to_keep=5) if load:
def main(): signal.signal(signal.SIGINT, signal_handler) root = logging.getLogger() root.setLevel(logging.WARNING) root.addHandler(ColorizingStreamHandler()) parser = OptionParser(usage="\ %prog [options]\n\ Example 1: %prog --tag\n\ Example 2: %prog --tag -b --username='******' --interactive-password\n\ \ttag everything in the library, and authenticate to Plex Home as user [email protected] with password being prompted for (password can also be supplied using the --password option)\n\ Example 3: %prog -bq --tag --remove-all-tags --optimize --export-subtitles --embed-subtitles -ip 192.168.0.2 --port 55400\n\ Example 4: %prog --subtitles -m 'D:\Movies' '/Volumes/Media/Movies' -m '\\' '/'\n\ Example 5: %prog -tb --batch-mediatype=movie --batch-breadcrumb='kids>cars'\n\ \tonly tag movies who are in a section containing the word 'kids' and movies who's name contains 'cars'\n\ Example 6: %prog -tb --batch-mediatype=show --batch-breadcrumb='>lost>season 1>pilot'\n\ \tonly tag tv episodes, matches all sections, show name contains lost, season 1, episode title contains 'pilot'\n\ Example 7: %prog -tb --batch-breadcrumb='tv>weeds>>goat'\n\ \tonly tag items who are in a section who's title contains 'tv', where the movie or show name contains 'weeds', any season and episode title contains 'goat' \n\ %prog -h for full list of options\n\n\ Filepaths to media items in PMS need to be the same as on machine that is running this script (can be worked around by using the -m option to modify the file path).\ ") parser.add_option( "-t", "--tag", action="store_true", dest="tag", help="tag all compatible file types, and update any previously tagged files (if metadata in plex has changed)") parser.add_option( "--tag-update", action="store_true", dest="tag_update", help="update previously tagged files if the PMS entry has changed since last time (modification time)") parser.add_option( "--tag-tv-prefer-season-artwork", action="store_true", dest="tag_prefer_season_artwork", help="when tagging tv show episodes, the season artwork will be used instead of the episode thumbnail") parser.add_option( "-r", "--remove-tags", action="store_true", dest="removetags", help="remove all compatible tags from the files") parser.add_option( "-f", "--force", action="store_true", dest="force", help="ignore previous work and steam ahead with task (will re-tag previously tagged files, re-enters data into TV, etc.)") parser.add_option( "-o", "--optimize", action="store_true", dest="optimize", help="interleave the audio and video samples, and put the \"MooV\" atom at the beginning of the file") parser.add_option( "--chapter-previews", action="store_true", dest="chapter_previews", help="generate preview images for any chapter markers") parser.add_option( "--export-subtitles", action="store_true", dest="export_subtitles", help="export any subtitles to the same path as the video file") parser.add_option( "--embed-subtitles", action="store_true", dest="embed_subtitles", help="embed compatible files with a compatible \"sidecar\" subtitle file if present") parser.add_option( "--export-artwork", action="store_true", dest="export_artwork", help="export the artwork to the same path as the video file") parser.add_option( "--stats", action="store_true", dest="gather_statistics", help="gather \"interesting\" statistics about the items being processed") parser.add_option( "-m", action="append", type="string", dest="path_modifications", nargs=2, metavar="<find> <replace>", help="perform a find & replace operation on the pms' media file paths (useful if you are running the script on a different machine than the one who is hosting the pms, i.e. the mount paths are different). Supply multiple times to perform several different replacements (operations are performed in order supplied).") parser.add_option( "--open", action="store_true", dest="open_file_location", help="open a Finder window at the containing folder of the file just processed (Mac OS X only)") parser.add_option( "--add-to-tv", action="store_true", dest="add_to_tv", help="adds the item to TV if not already present") parser.add_option( "-i", "--ip", action="store", dest="ip", type="string", help="specify an alternate IP address that hosts a PMS to connect to (default is localhost)") parser.add_option( "-p", "--port", action="store", dest="port", type="int", help="specify an alternate port number to use when connecting to the PMS (default is 32400)") parser.add_option( "--username", action="store", dest="username", type="string", help="specify the username to use when authenticating with the PMS (default is no authentication)") parser.add_option( "--password", action="store", dest="password", type="string", help="specify the password to use when authenticating with the PMS (default is no authentication)") parser.add_option( "--interactive-password", action="store_true", dest="interactive_password", help="the password to use when authenticating with the PMS will be supplied interactively") parser.add_option( "--interactive", action="store_true", dest="interactive", help="interactivly select files to operate on [default]") parser.add_option( "-b", "--batch", action="store_false", dest="interactive", help="disable interactive mode. Requires no human intervention once launched, and will perform operations on all valid files") parser.add_option( "--batch-mediatype", action="store", dest="batch_mediatype", type="choice", choices=["any", "movie", "show"], metavar="[movie/show]", help="only specified media type will be processed") parser.add_option( "--batch-breadcrumb", action="store", dest="batch_breadcrumb", type="string", metavar="breadcrumb", help="only items matching the breadcrumb trail will be processed. Components seperated by '>' (case insensitive)") parser.add_option( "-v", "--verbose", dest="verbose", action="callback", callback=setLogLevel, help='increase verbosity (can be supplied 0-2 times)') parser.add_option( "-q", "--quiet", action="store_true", dest="quiet", help="ninja-like processing (can only be used when in batch mode)") parser.add_option( "-d", "--dry-run", action="store_true", dest="dryrun", help="pretend to do the job, but never actually change or export anything. Pretends that all tasks succeed. Useful for testing purposes") parser.set_defaults( tag=False, tag_update=False, tag_prefer_season_artwork=False, remove_tags=False, optimize=False, chapter_previews=False, embed_subtitles=False, export_resources=False, export_subtitles=False, export_artwork=False, gather_statistics=False, open_file_location=False, add_to_tv=False, force_tagging=False, dryrun=False, interactive=True, quiet=False, batch_mediatype="any", batch_breadcrumb="", ip="localhost", port=32400, username="", password="", interactive_password=False, path_modifications=[]) try: opts, args = parser.parse_args() except OptionValueError as e: parser.error(e) if opts.export_subtitles or opts.export_artwork: opts.export_resources = True if not opts.tag and not opts.removetags and not opts.optimize and not opts.export_resources and not opts.add_to_tv and not opts.gather_statistics: parser.error("No task to perform. Our work here is done...") if opts.tag_prefer_season_artwork and not opts.tag: parser.error("Cannot prefer season artwork when not tagging...") if opts.chapter_previews and not opts.tag: parser.error("Cannot generate chapter previews when not tagging...") if opts.embed_subtitles and not opts.tag: parser.error("Cannot embed subtitles when not tagging...") if opts.tag_update and not opts.tag: parser.error("Cannot update tags when not tagging...") if opts.interactive and ( opts.batch_mediatype != "any" or len(opts.batch_breadcrumb) > 0): parser.error("Cannot use batch filtering options when batch mode is not active...") if opts.interactive_password: opts.password = getpass.getpass("Password $") if (len(opts.username) > 0 and not len(opts.password) > 0) or (len(opts.password) > 0 and not len(opts.username) > 0): parser.error("Must supply both username and password when using authentication to connect to PMS...") if len(opts.batch_breadcrumb) > 0: opts.batch_breadcrumb = opts.batch_breadcrumb.lower().split(">") opts.batch_breadcrumb.reverse() if opts.quiet: root.setLevel(logging.ERROR) if opts.interactive and not root.isEnabledFor(logging.INFO): root.setLevel(logging.INFO) if opts.dryrun: logging.critical( "WARNING, RUNNING IN 'DRY RUN MODE'. NO ACTUAL CHANGES WILL BE MADE" ) elif opts.removetags: logging.critical( "WARNING, TAGS WILL BE REMOVED PERMANENTLY" ) elif opts.force: logging.critical( "FORCE MODE ENABLED. THIS WILL BYPASS ANY 'HAS THIS BEEN DONE BEFORE' CHECKS" ) logging.error( generate_centered_padded_string(" Plex Media Tagger Started ") ) if opts.gather_statistics: statistics = LibraryStatistics() summary = Summary() request_handler = PmsRequestHandler() request_handler.ip = opts.ip request_handler.port = opts.port request_handler.setup_opener(opts.username, opts.password) global section_processor section_processor = SectionProcessor(opts, request_handler) logging.error( "Connecting to PMS at %s:%d" % (opts.ip, opts.port) ) sections_container = request_handler.get_sections_container() media_container = sections_container.getroot() title = media_container.attrib['title1'] section_elements = media_container.getchildren() section_element_choice = '' #default is empty == all if opts.interactive: logging.info( "List of sections for %s" % title ) for index, section_element in enumerate(section_elements): logging.info( "%d. %s" %(index, section_element.attrib['title']) ) #end for if len(section_elements) == 0: logging.error( "No sections found" ) else: logging.warning( "empty input equals all" ) #ask user what sections should be processed section_element_choice = raw_input("Section to process $") if section_element_choice != '': try: section_element_choice = int(section_element_choice) except ValueError, e: logging.debug(e) logging.critical( "'%s' is not a valid section number" % input ) sys.exit(1)
def addToSummary(self, summary: Summary, zone, month, year): excelFileName = "Monthly Report "+zone.upper()+" " + \ "01-"+"{:0>2d}".format(int(month))+"-"+year+".xlsx" monthFullName = datetime.datetime.now().strftime("%B") fileName = 'User Data/' + monthFullName.upper( ) + "_" + year + "/" + excelFileName print(fileName) # return workbook = load_workbook(filename=fileName) activeSheet = workbook['Bill Summary'] # activeSheet = workbook.active # inserting and formatting rows noOfRows = len(summary.getRecordsList()) - 3 if noOfRows > 0: activeSheet.insert_rows(11, noOfRows) formattedrow = list(activeSheet.rows)[9] print(formattedrow[0]) # unformattedrow = list(worksheet.rows)[9] for row in range(noOfRows): unformattedrow = list(activeSheet.rows)[10 + row] print(unformattedrow[0]) for (c1, c2) in zip(formattedrow, unformattedrow): c2.style = copy(c1.style) c2.border = copy(c1.border) c2.number_format = copy(c1.number_format) c2.font = copy(c1.font) c2.alignment = copy(c1.alignment) c2.fill = copy(c1.fill) # inserting data into summary records = summary.getRecordsList() totalRecords = len(summary.getRecordsList()) record = 0 fromRow = 10 toRow = fromRow + totalRecords i = 0 for i in range(fromRow, toRow): print(records[record][0]) activeSheet['A' + str(i)] = record + 1 activeSheet['B' + str(i)] = records[record][0] activeSheet['C' + str(i)] = "[" + str(records[record][1]) + "]" activeSheet['D' + str(i)] = records[record][2] activeSheet['E' + str(i)] = records[record][3] activeSheet['F' + str(i)] = records[record][4] activeSheet['G'+str(i)] = records[record][3] + \ records[record][4] activeSheet['H' + str(i)] = records[record][5] activeSheet['J' + str(i)] = records[record][6] record = record + 1 # Date Cell activeSheet['A5'] = " Bank Al Habib Limited Date:" + \ "01-"+"{:0>2d}".format(int(month))+"-"+year # Zone Cell activeSheet['A6'] = "Bills Summary " + zone + " ZONE" # updating formulas formulaCells = ['E', 'F', 'G', 'H', 'J'] if noOfRows > 0: for formulaCell in formulaCells: activeSheet[formulaCell+str(toRow)] = "=SUM(" + \ formulaCell+str(fromRow)+":"+str(toRow-1)+")" workbook.save(filename=fileName)
def export_resources(self, part_item): part_item_file_path = part_item.modified_file_path() directory = os.path.dirname(part_item_file_path) filename = os.path.basename(part_item_file_path) filename_without_extension = os.path.splitext(filename)[0] #=== subtitles === #build up language_code dict if self.opts.export_subtitles: logging.warning("attempting to export subtitles...") subtitle_stream_type = "3" all_non_embedded_subtitles = [] for stream_item in part_item.stream_items: if stream_item.stream_type == subtitle_stream_type and stream_item.key != "": all_non_embedded_subtitles.append(stream_item) #end if #end for number_of_non_embedded_subtitles = len(all_non_embedded_subtitles) if number_of_non_embedded_subtitles == 0: logging.warning("no subtitles found") else: logging.warning("found %d subtitle(s)" % number_of_non_embedded_subtitles) categorized_subtitles = {} for subtitle in all_non_embedded_subtitles: key = (subtitle.language_code, subtitle.codec) if categorized_subtitles.has_key(key): categorized_subtitles[key].append(subtitle) else: categorized_subtitles[key] = [subtitle] #end if has_key #end for all_subtitles exported_subtitles = 0 for key, subtitles in categorized_subtitles.iteritems(): #key = (eng, srt), (eng, sub), (fre, srt), etc. language_code = key[0] codec = key[1] #get all existing sub files. example filename: Sopranos - S01E01 - The Pilot*.eng.srt compatible_sidecar_subtitles = self.get_all_sidecar_subtitles( directory, filename_without_extension, language_code, codec) if len(compatible_sidecar_subtitles) > 0: logging.warning( "Subtitle file(s) with language code '%s' of type '%s' already exist. Skipping all matching..." % (language_code, codec)) continue #end if #export subs i = 0 for subtitle in subtitles: if i == 0: subtitle_filename = "%s.%s.%s" % ( filename_without_extension, language_code, codec) else: subtitle_filename = "%s.%02d.%s.%s" % ( filename_without_extension, i, language_code, codec) #end if subtitle_full_path = os.path.join( directory, subtitle_filename) success = subtitle.export_to_path(subtitle_full_path) if success: i += 1 exported_subtitles += 1 Summary().subtitle_export_succeeded() else: Summary().subtitle_export_failed() #end success #end for subtitles #end for categorized_subtitles logging.warning( "exported %d subtitle(s), skipped %d" % (exported_subtitles, (number_of_non_embedded_subtitles - exported_subtitles))) #end if len(all_non_embedded_subtitles) == 0: #end if subtitles if self.opts.export_artwork: logging.error("artwork export not yet implemented...")
def Backtest(self): ''' temp_df : accept DataFrame with columen:CLOSE HIGH LOW OPEN VOLUME index : datetime object Stops=10 : int : number of tick to hit stop from entry price Troubleshoot=False : Boolean : setting to True will also return the source dataframe along with the summary TP_list=[20,30] : list of length 2 : a pair of targets for from the entry calculated in number of ticks, to set a single target , put leave the TP_list[1] empty and set TP_size to 1 TP_size=0.5 : float : The proportion of current total position to exit when TP_list[0] reach, set to 1 if exiting all at TP_list[0] tick_size=0.5: float : the smallest move available in the asset Slippage=1 : int : the number of ticks to acct of slippage when hitting the Stops tier_size=1 : int : the number of assset to buy on each signal CAGR_N=1 : flaot : in number of years, eg 6month ==0.5 ''' self.temp_df['prev_sign'] = self.temp_df['sign'].shift() self.temp_df['periods'] = np.nan self.temp_df['EntryP'] = np.nan self.temp_df['Stop'] = np.nan self.temp_df['TPs'] = np.nan EntryPrice, ExitPrice, Position = initialise_setting() self.temp_df['PL'] = 0 # Loop Thru the Dataframe for i in range(self.temp_df.shape[0]): print(f"Row: {i}/{self.temp_df.shape[0]-1}", end='\r') self.PL = 0 Close = self.temp_df.iloc[ i, self.temp_df.columns.str.upper().get_loc("CLOSE")] High = self.temp_df.iloc[ i, self.temp_df.columns.str.upper().get_loc("HIGH")] Low = self.temp_df.iloc[ i, self.temp_df.columns.str.upper().get_loc("LOW")] OPEN = self.temp_df.iloc[ i, self.temp_df.columns.str.upper().get_loc("OPEN")] ###########################################SL and TP check########################################## # Assume all position exited with a slippage*tick_size # as we may not know which direction it will go first, stop out is chosen to be the conservative first direction ############################################## SL ###################################### # Stop_hit(ExitPrice,EntryPrice,Low,High,Position) # - if Stops hit # - LowExitPrice,EntryPrice,,High,Position will be re-initialized # - self.PL = Position * (ExitPrice['Stop'] - EntryPrice) - self.Slippage # Else it will return the input value for ExitPrice,EntryPrice,Low,High,Position EntryPrice, ExitPrice, Position = self.Stop_Hit( ExitPrice, EntryPrice, Low, High, Position) ################################################ TP ############################################### # TP_hit(ExitPrice,EntryPrice,Low,High,Position) # - if Stops hit # - LowExitPrice,EntryPrice,,High,Position will be re-initialized # - self.PL = Position * (ExitPrice['Stop'] - EntryPrice) - self.Slippage # Else it will return the input value for ExitPrice,EntryPrice,Low,High,Position EntryPrice, ExitPrice, Position = self.TP_hit( ExitPrice, EntryPrice, Low, High, Position) EntryPrice, ExitPrice, Position = self.New_position( i, ExitPrice, EntryPrice, Position, Close) # CLOSE ALL at last period if i == self.temp_df.shape[0] - 1: if Position != 0: self.PL = Position * (Close - EntryPrice) # Append the PL for this Epoch to the List later to become the PL column self.PL_list.append(self.PL) ######################## Outside for loop calculate summaries ############################################### self.temp_df['PL'] = self.PL_list self.temp_df['Cumm_PL'] = self.temp_df['PL'].cumsum() # adding to summary data to a Dataframe Summary_dict = S.summary(self.temp_df.PL, ref_price=self.temp_df['OPEN'].iloc[0], n=self.CAGR_N).Summary_dict Summary_dict['Period'] = 'Start: ' + str( self.temp_df.index[0]) + ' End: ' + str(self.temp_df.index[-1]) Summary_dict['NO. of Buy signal'] = self.temp_df['Actions'][ self.temp_df['Actions'] == 'BUY'].count() Summary_dict['NO. of Sell signal'] = self.temp_df['Actions'][ self.temp_df['Actions'] == 'SELL'].count() #print(Summary_dict) if self.Troubleshoot == True or self.Return_df == True: return [Summary_dict, self.temp_df] return Summary_dict
def run(self): """Thread entry point""" sRunError = "" self.userSourceIDs = True log.debug("Running Sequence") try: # Main sequence run loop log.debug("sourceSequence -> %s" % str(self.sourceSequence)) sMessage = "Run of sequence \"" + self.sourceSequence["metadata"]["name"] + "\" started." log.debug(sMessage) self.messaging.broadcastMessage(sMessage) nComponentCount = len(self.sourceSequence["components"]) nCurrentComponent = 0 while nCurrentComponent < nComponentCount: # Get the next component pSourceComponent = self.sourceSequence["components"][nCurrentComponent] # Skip components until we find our start component self.sourceComponentID = pSourceComponent["id"] if self.initializing and (self.startComponentID != 0) and (self.sourceComponentID != self.startComponentID): log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")") nCurrentComponent += 1 continue # Update our initializing and running flags self.initializing = False self.running = True # Ignore any previous summary component if pSourceComponent["componenttype"] == Summary.componentType: log.debug("Skipping unit operation (" + pSourceComponent["componenttype"] + ")") nCurrentComponent += 1 continue # Create and run the next unit operation log.debug("Starting unit operation " + str(self.sourceComponentID) + " (" + pSourceComponent["componenttype"] + ")") pSourceUnitOperation = UnitOperations.createFromComponent(self.sourceSequenceID, pSourceComponent, self.username, self.sequenceManager.database, self.systemModel) self.runComponentID = pSourceUnitOperation.copyComponent(self.sourceSequenceID, self.runSequenceID) pRunComponent = self.sequenceManager.GetComponent(self.username, self.runComponentID, self.runSequenceID) pRunUnitOperation = UnitOperations.createFromComponent(self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel) pRunUnitOperation.setDaemon(True) pRunUnitOperation.start() self.systemModel.SetUnitOperation(pRunUnitOperation) log.debug( "Unit Op is alive") # Wait until the operation completes or we receive an abort signal while pRunUnitOperation.is_alive() and not self.runAborted: log.debug( "Unit Op check alive") time.sleep(0.25) self.systemModel.SetUnitOperation(None) if self.runAborted: log.error( "Unit Op ABORTED!") pRunUnitOperation.setAbort() raise Exception("Run aborted") # Check for unit operation error log.debug( "Unit check for error") sError = pRunUnitOperation.getError() if sError != "": log.error("UnitOperation: %s" % pRunUnitOperation.__class__.__name__) log.error( "Unit operation failed: " + sError) raise Exception(sError) log.debug("Prepare to update operation details") # Update the unit operation details in the database UnitOperations.updateToComponent(pRunUnitOperation, self.runSequenceID, pRunComponent, self.username, self.sequenceManager.database, self.systemModel) log.debug("After updateToComponent") self.sequenceManager.UpdateComponent(self.username, self.runSequenceID, self.runComponentID, None, pRunComponent) log.debug("After updateComponent") log.debug("Completed unit operation (" + pRunComponent["componenttype"] + ")") self.sourceComponentID = 0 self.runComponentID = 0 # Check if the user paused the sequence for editing if self.runWillPause: # Pause until editing is complete log.debug("Pausing run for editing") self.runWillPause = False self.runIsPaused = True while self.runIsPaused and not self.runAborted: time.sleep(0.25) if self.runAborted: raise Exception("Sequence aborted") log.debug("Continuing paused run") # Reload the sequence and make sure it is still valid self.sourceSequence = self.sequenceManager.GetSequence(self.username, self.sourceSequenceID) if not self.sourceSequence["metadata"]["valid"]: raise Exceptions("Edited sequence is no longer valid (" + self.sourceSequenceID + ")") # Scan until we find the unit operation we just executed nComponentCount = len(self.sourceSequence["components"]) nCurrentComponent = 0 while nCurrentComponent < nComponentCount: if self.sourceSequence["components"][nCurrentComponent]["id"] == pSourceComponent["id"]: break nCurrentComponent += 1 if nCurrentComponent == nComponentCount: raise Exception("Failed to find previous component in edited sequence") # Advance to the next component nCurrentComponent += 1 except Exception as ex: log.error("Sequence run failed: " + str(ex)) log.error("Trace Back %s" % traceback.format_exc()) sRunError = str(ex) # Add the Summary unit operation to the sequence pSummaryComponent = Summary.createNewComponent(sRunError) self.runComponentID = self.database.CreateComponent(self.username, self.runSequenceID, pSummaryComponent["type"], pSummaryComponent["note"], json.dumps(pSummaryComponent)) # Fully validate the run sequence self.sequenceManager.ValidateSequenceFull(self.username, self.runSequenceID) # Switch to using the run IDs rather than the source because the summary unit operation only exists in the run sequence self.userSourceIDs = False # Instantiate and start the summary unit operation log.info("Starting summary unit operation") pSummaryComponent = self.sequenceManager.GetComponent(self.username, self.runComponentID, self.runSequenceID) pSummaryUnitOperation = UnitOperations.createFromComponent(self.runSequenceID, pSummaryComponent, self.username, self.sequenceManager.database, self.systemModel) pSummaryUnitOperation.setDaemon(True) pSummaryUnitOperation.start() self.systemModel.SetUnitOperation(pSummaryUnitOperation) # Broadcast the summary unit operation message self.messaging.broadcastMessage(pSummaryComponent["message"]) # Wait until the operation completes while pSummaryUnitOperation.is_alive(): time.sleep(0.25) log.info("Summary unit operation complete") self.runComponentID = 0 # Run complete log.debug("Run stopped") self.running = False
def upload_file(): try: if 'file' not in request.files: flash('No file part') return redirect(request.url) uploaded_files = request.files.getlist("file") milliseconds = request.args.get("milliseconds") rules = request.args.get("rules") rules = rules.split(',') link = 'https://s3.us-east.cloud-object-storage.appdomain.cloud/sharad-saurav-bucket/DataFiles_Rules_Report.xlsx' target, headers = urllib.request.urlretrieve(link) print(target, headers) newTar = target + ".xlsx" os.rename(target, newTar) length = len(rules) numberOfFiles = len(uploaded_files) fileNames = [] for file in uploaded_files: fileNames.append(file.filename) for file in uploaded_files: if file.filename == '': flash('No selected file') return redirect(request.url) if file and allowed_file(file.filename): filename = secure_filename(file.filename) Check_Columns.checkColumn(file, filename) if ("Allowed_intents_in_Unstructured" in rules): Allowed_intents_in_Unstructured.rule_unstructured( file, filename, newTar) if ("Check_for_duplicates" in rules): Check_for_duplicates.rule_duplicates( file, filename, newTar) if ("Check_for_missing_Keyword" in rules): Check_for_missing_Keyword.rule_missing_keyword( file, filename, newTar) if ("Check_for_capitalization" in rules): Check_for_capitalization.rule_capitalization( file, filename, newTar) if ("Check_if_date_time_are_blank" in rules): Check_if_date_time_are_blank.rule_date_time_blank( file, filename, newTar) if ("Correctness_of_MAP_URL" in rules): Correctness_of_MAP_URL.rule_map_url(file, filename, newTar) if ("Correctness_of_Short_Ref_URL" in rules): Correctness_of_Short_Ref_URL.short_ref_url( file, filename, newTar) if ("Date_in_YYYY_MM_DD_format" in rules): Date_in_YYYY_MM_DD_format.date_format( file, filename, newTar) if ("Description_text_not_same" in rules): Description_text_not_same.description_text( file, filename, newTar) if ("Duplicate_in_Entity_Interactn" in rules): Duplicate_in_Entity_Interactn.duplicate_entity_interaction( file, filename, newTar) if ("Email_id_validity" in rules): Email_id_validity.email_id_validity(file, filename, newTar) if ("Exact_dates_available" in rules): Exact_dates_available.exact_dates_available( file, filename, newTar) if ("Exceeding_500_characters" in rules): Exceeding_500_characters.exceeding_500_characters( file, filename, newTar) if ("Latitude_Longitude" in rules): Latitude_Longitude.latitide_longitude( file, filename, newTar) if ("Multiple_Spaces_in_txt" in rules): Multiple_Spaces_in_txt.multiple_spaces_in_txt( file, filename, newTar) if ("No_content_in_brackets" in rules): No_content_in_brackets.no_content_in_brackets( file, filename, newTar) if ("No_AcadEvents_in_Timing" in rules): No_AcadEvents_in_Timing.no_acadEvents_in_timing( file, filename, newTar) if ("No_date_special_characters" in rules): No_date_special_characters.No_date_special_characters( file, filename, newTar) if ("No_phone_url_in_voice" in rules): No_phone_url_in_voice.no_phone_url_in_voice( file, filename, newTar) if ("No_preceeding_0_in_room_no" in rules): No_preceeding_0_in_room_no.no_preceeding_0_in_room_no( file, filename, newTar) if ("No_Ref_URL_in_text" in rules): No_Ref_URL_in_text.no_ref_url_in_text( file, filename, newTar) if ("No_sentence_in_voice_column" in rules): # No_sentence_in_virtual_entity.no_sentence_in_virtual_entity(file, filename, newTar) No_sentence_in_voice_column.no_sentence_in_voice_column( file, filename, newTar) if ("No_timing_for_acad_events" in rules): No_timing_for_acad_events.no_timing_for_acad_events( file, filename, newTar) if ("No_timings_values_in_txt" in rules): No_timings_values_in_txt.no_timings_values_in_txt( file, filename, newTar) if ("Numbering_bullet_points" in rules): Numbering_bullet_points.numbering_bullet_points( file, filename, newTar) if ("Process_ID" in rules): # Perfect_Excel_format.perfect_excel_format(file, filename, newTar) Process_ID.process_id(file, filename, newTar) if ("Special_Char_in_Entity_Name" in rules): Special_Char_in_Entity_Name.special_char_in_entity_name( file, filename, newTar) if ("Start_date_less_than_end_date" in rules): Start_date_less_than_end_date.start_date_less_than_end_date( file, filename, newTar) if ("Start_time_less_than_end_time" in rules): Start_time_less_than_end_time.start_time_less_than_end_time( file, filename, newTar) if ("Time_in_HH_MM_SS_format" in rules): Time_in_HH_MM_SS_format.time_in_hh_mm_ss_format( file, filename, newTar) Summary.summary(newTar, numberOfFiles, rules, fileNames) uploadFile.multi_part_upload( "sharad-saurav-bucket", "DataFiles_Rules_Report" + milliseconds + ".xlsx", newTar) return getJson.get_Json_data(newTar, length) except Exception as e: traceback.print_exc() return str(e)