def run(self,max_wait_time=60*60): self.log_file='%s/Tracking/SlurmTracker.log'%my_path() self.pickle_file='%s/Tracking/SlurmTracker.pickle'%my_path() self.max_wait_time=max_wait_time time.sleep(10) while True: self.last_check=time.ctime() log=open(self.log_file,'w+') #Kill daemon when pidfile is erased self.checkpid() if not pid: log.write("Daemon killed remotely.\n") log.write("Daemon stop at %s\n"%time.ctime(time.time())) log.close() break #List jobs log.write("Job list at %s\n"%time.ctime(time.time())) self.listjobs() log.write('\n'.join(['\t'.join([str(j.jobid),j.state]) for j in self.joblist])+'\n') #Check every job self.checkjobs() if len(self.joblist)==0: break with open(self.pickle_file,'w+') as f: pickle.dump(self,f) log.write('Waiting %i seconds\n'%(self.wait_time+10)) log.close() time.sleep(10+self.wait_time) #If end near, check in 10s log=open(self.log_file,'a+') log.write("No more processes active, Daemon stop at %s\n"%time.ctime(time.time())) log.close() os.remove(self.pickle_file)
def __init__(self, datestart, datestop, n_dates, columns, statusfile=None, status_interval=None, outfile=None, max_size=None, ): self.statusfile= statusfile self.status_interval = status_interval self.outfile = outfile self.max_size = max_size self.total_rows = n_dates self.percent_complete = 0 self.row_interval = 100 # Check time every 100 rows self.last_time = 0.0 self.process_start = time.ctime() self.current_row = 0 self.current_time = time.ctime() self.datestart = Chandra.Time.DateTime(datestart).date self.datestop = Chandra.Time.DateTime(datestop).date self.columns = ' '.join([x.name for x in columns]) self.error = None self.filesize = 0 self.print_attrs = ('current_row', 'total_rows', 'percent_complete', 'process_start', 'current_time', 'datestart', 'datestop', 'columns', 'status', 'error')
def run_debugger(testname, pythonfile, pydb_opts='', args='', outfile=None): global srcdir, builddir, pydir rightfile = os.path.join(srcdir, 'data', "%s.right" % testname) os.environ['PYTHONPATH']=os.pathsep.join(sys.path) cmdfile = os.path.join(srcdir, "%s.cmd" % testname) outfile = "%s.out" % testname outfile_opt = '--output=%s ' % outfile # print "builddir: %s, cmdfile: %s, outfile: %s, rightfile: %s" % \ # (builddir, cmdfile, outfile, rightfile) if os.path.exists(outfile): os.unlink(outfile) cmd = "%s --command %s %s %s %s %s" % \ (pydb_path, cmdfile, outfile_opt, pydb_opts, pythonfile, args) os.system(cmd) fromfile = rightfile fromdate = time.ctime(os.stat(fromfile).st_mtime) fromlines = open(fromfile, 'U').readlines() tofile = outfile todate = time.ctime(os.stat(tofile).st_mtime) tolines = open(tofile, 'U').readlines() diff = list(difflib.unified_diff(fromlines, tolines, fromfile, tofile, fromdate, todate)) if len(diff) == 0: os.unlink(outfile) for line in diff: print line, return len(diff) == 0
def main(): time1 = time.time() print time.ctime() get_column_urls() # use 5 threads to get article_urls for i in xrange(WORKER_THREAD_NUM): producer_thread = threading.Thread(target=get_article_urls) producer_thread.setDaemon(True) producer_thread.start() producer_threads.append(producer_thread) for producer_thread in producer_threads: producer_thread.join() # after getting all article_urls , start 'worker' for i in xrange(WORKER_THREAD_NUM): thread = threading.Thread(target=worker) thread.setDaemon(True) # need this?? thread.start() threads.append(thread) for thread in threads: thread.join() time2 = time.time() print 'finised in %s seconds' % (time2-time1)
def get(self): settings = get_server_settings() secret = self.request.headers.get("X-Nuntiuz-Secret", None) if secret != settings.jabberSecret: logging.error("Received unauthenticated apple certificate request, ignoring ...") return app_id = self.request.get("id") if not app_id: return app = get_app_by_id(app_id) if not app: return if app.apple_push_cert_valid_until < now() + 30 * DAY: send_mail(settings.dashboardEmail, settings.supportWorkers, "The APN cert of %s is about to expire" % app_id, "The APN cert of %s is valid until %s GMT" % (app_id, time.ctime(app.apple_push_cert_valid_until))) if app.apple_push_cert_valid_until < now() + 15 * DAY: logging.error("The APN cert of %s is valid until %s GMT" % (app_id, time.ctime(app.apple_push_cert_valid_until))) result = json.dumps(dict(cert=app.apple_push_cert, key=app.apple_push_key, valid_until=app.apple_push_cert_valid_until)) self.response.headers['Content-Type'] = 'application/binary' _, data = encrypt_for_jabber_cloud(secret, result) self.response.write(data)
def cmd_time(args): '''show autopilot time''' tusec = mpstate.master().field('SYSTEM_TIME', 'time_unix_usec', 0) if tusec == 0: print("No SYSTEM_TIME time available") return print("%s (%s)\n" % (time.ctime(tusec * 1.0e-6), time.ctime()))
def func(args): # Get modification times fromdate = time.ctime(os.stat(args.FILE1).st_mtime) todate = time.ctime(os.stat(args.FILE2).st_mtime) # Open fromfile try: with open(args.FILE1, 'U') as fd: fromlines = fd.readlines() except IOError: print("Error opening file " + args.FILE1, file=sys.stderr) # Open tofile try: with open(args.FILE2, 'U') as fd: tolines = fd.readlines() except IOError: print("Error opening file " + args.FILE2, file=sys.stderr) # Create diff if args.unified: diff = difflib.unified_diff(fromlines, tolines, args.FILE1, args.FILE2, fromdate, todate, n=args.lines) elif args.ndiff: diff = difflib.ndiff(fromlines, tolines) elif args.html: diff = difflib.HtmlDiff().make_file(fromlines, tolines, args.FILE1, args.FILE2, context=args.context, numlines=args.lines) else: diff = difflib.context_diff(fromlines, tolines, args.FILE1, args.FILE2, fromdate, todate, n=args.lines) # we're using writelines because diff is a generator sys.stdout.writelines(diff)
def time_cuo(): # 10位 print(int(time.time())) # 13位 # 1521204495005 print(int(time.time()*1000)) #十三位转为时间 t_1= 1540544495276/1000.00 print('t_1', time.ctime(t_1)) t_2= 1533825084 print('t_2', time.ctime(t_2)) t_s=1540544411037/1000.0 t2 = 1503048528 * 1.00 updateTime = 1504602551076 / 1000.00 #t3 = time.ctime(t) t4 = time.ctime(t2) t5=time.ctime(t_s) print('t5',t5) #print('t3', t3) print('t4', t4) curr_d = datetime.datetime.fromtimestamp(updateTime) print('curr_d',curr_d) d_time = datetime.datetime.fromtimestamp(t_s) print(d_time)
def refresh_data(): global metrics global targets_all global graphs_all logger.debug('refresh_data() start') try: stat_metrics = backend.stat_metrics() if last_update is not None and stat_metrics.st_mtime < last_update: nice_metrics_mtime = time.ctime(stat_metrics.st_mtime) nice_last_update = time.ctime(last_update) logger.debug('refresh_data() not needed. metrics.json mtime %s, last_update %s', nice_metrics_mtime, nice_last_update) return ("metrics.json is last updated %s, " "rebuilding data structures wouldn't make sense cause they were " "last rebuilt %s" % (nice_metrics_mtime, nice_last_update)) (metrics, targets_all, graphs_all) = backend.update_data(s_metrics) if 'metrics_file' in errors: del errors['metrics_file'] logger.debug('refresh_data() end ok') return 'ok' except MetricsError, e: errors['metrics_file'] = (e.msg, e.underlying_error) logger.error("[%s] %s", e.msg, e.underlying_error) logger.error('refresh_data() failed') response.status = 500 return "errors: %s" % ' '.join('[%s] %s' % (k, v) for (k, v) in errors.items())
def get_repository_info(recipe_path): """This tries to get information about where a recipe came from. This is different from the source - you can have a recipe in svn that gets source via git.""" try: if exists(join(recipe_path, ".git")): origin = check_output_env(["git", "config", "--get", "remote.origin.url"], cwd=recipe_path) rev = check_output_env(["git", "rev-parse", "HEAD"], cwd=recipe_path) return "Origin {}, commit {}".format(origin, rev) elif isdir(join(recipe_path, ".hg")): origin = check_output_env(["hg", "paths", "default"], cwd=recipe_path) rev = check_output_env(["hg", "id"], cwd=recipe_path).split()[0] return "Origin {}, commit {}".format(origin, rev) elif isdir(join(recipe_path, ".svn")): info = check_output_env(["svn", "info"], cwd=recipe_path) server = re.search("Repository Root: (.*)$", info, flags=re.M).group(1) revision = re.search("Revision: (.*)$", info, flags=re.M).group(1) return "{}, Revision {}".format(server, revision) else: return "{}, last modified {}".format(recipe_path, time.ctime(os.path.getmtime( join(recipe_path, "meta.yaml")))) except CalledProcessError: log.debug("Failed to checkout source in " + recipe_path) return "{}, last modified {}".format(recipe_path, time.ctime(os.path.getmtime( join(recipe_path, "meta.yaml"))))
def test2(): now = time.strftime("%Y-%m-%d") print(now) print(type(now)) now_time = datetime.datetime.now() print(now_time) print(type(now_time)) today = time.strftime("_%Y_%m_%d") print(today) aa = datetime.datetime(2016, 8, 7) # 看输入的日期是一个星期的第几天 print(aa.weekday()) print(str(int(time.time() * 1000))) t1 = time.time() print(t1) print(time.ctime()) # t = 1490060308998 * 1.00 / 1000 t = 1503048528 * 1.00 print(time.ctime(t1)) print(time.ctime(t)) t3 = time.ctime(t) print(time.gmtime(t)) # print(time.strptime(t3,'%Y-%m-%d')) print(type(t3))
def checkFilesChanged(self, files, unchanged_for = 60): now = time.time() file_too_new = False file_time = [] for cur_file in files: # File got removed while checking if not os.path.isfile(cur_file): file_too_new = now break # File has changed in last 60 seconds file_time = self.getFileTimes(cur_file) for t in file_time: if t > now - unchanged_for: file_too_new = tryInt(time.time() - t) break if file_too_new: break if file_too_new: try: time_string = time.ctime(file_time[0]) except: try: time_string = time.ctime(file_time[1]) except: time_string = 'unknown' return file_too_new, time_string return False, None
def time_diff(self, session, clock="sys"): """ Calculate system/hardware time difference between host and guest; :param session: ShellSession object; :param clock: 'hw'or 'sys' :return: time difference :rtype: float """ time_command = self.params.get("%s_time_command" % clock) time_filter_re = self.params.get("%s_time_filter_re" % clock) time_format = self.params.get("%s_time_format" % clock) host_time, guest_time = self.get_time(session, time_command, time_filter_re, time_format) guest_ctime = time.ctime(guest_time) host_ctime = time.ctime(host_time) debug_msg = "Host %s Time: %s = %s epoch seconds " % (clock, host_ctime, host_time) logging.info(debug_msg) debug_msg = "Guest %s Time: %s = %s epoch seconds" % (clock, guest_ctime, guest_time) logging.info(debug_msg) return abs(host_time - guest_time)
def timestamp ( ): #*****************************************************************************80 # ## TIMESTAMP prints the date as a timestamp. # # Licensing: # # This code is distributed under the GNU LGPL license. # # Modified: # # 06 April 2013 # # Author: # # John Burkardt # # Parameters: # # None # import time t = time.time ( ) print time.ctime ( t ) return None
def Socket_TestNext(ippool): global Flag, Wait_for_SSL, Succ, InProcess, isEnd Lock.acquire() InProcess.append(1) Lock.release() while not isEnd: try: Lock.acquire() ip = ippool.pop(random.randrange(len(ippool))) Lock.release() if len(ippool)%1000 == 0: print len(Succ),"/",len(ippool), "/",len(Wait_for_SSL),"/",len(InProcess) print time.ctime(time.time()) #print time.time() Soc = Socket_Test.Socket_Test(ip) if Soc[0] == True: Lock.acquire() Wait_for_SSL.append(Soc) Lock.release() gevent.sleep() except KeyboardInterrupt: isEnd = True return 0 except (ValueError , IndexError): Lock.release() Flag = True Lock.acquire() InProcess.pop() Lock.release() return 0 Lock.acquire() InProcess.pop() Lock.release() return 0
def _process_demands(self): demands = self.demands repos = self.base.repos if not demands.cacheonly: if demands.freshest_metadata: for repo in repos.iter_enabled(): repo.md_expire_cache() elif not demands.fresh_metadata: for repo in repos.values(): repo.md_lazy = True if demands.sack_activation: lar = self.demands.available_repos self.base.fill_sack(load_system_repo='auto', load_available_repos=lar) if lar: repos = list(self.base.repos.iter_enabled()) if repos: mts = max(repo.metadata.timestamp for repo in repos) # do not bother users with fractions of seconds age = int(min(repo.metadata.age for repo in repos)) for repo in repos: logger.debug(_("%s: using metadata from %s."), repo.id, time.ctime(repo.metadata.md_timestamp)) if age != 0: logger.info(_("Last metadata expiration check: " "%s ago on %s."), datetime.timedelta(seconds=age), time.ctime(mts)) self.base.plugins.run_sack()
def get(self): """Responds to a single request for performance counter data. Each request has two required parameters in the query string, 'start' and 'end', which specify the beginning and end of the time range to be queried. The times should be expressed as the number of seconds since the unix epoch. """ start_time = float(self.get_argument('start')) end_time = float(self.get_argument('end')) # Select an appropriate interval resolution based on the requested time span. selected_interval = metric.LOGS_INTERVALS[-1] group_key = metric.Metric.EncodeGroupKey(metric.LOGS_STATS_NAME, selected_interval) logging.info('Query performance counters %s, range: %s - %s, resolution: %s' % (group_key, time.ctime(start_time), time.ctime(end_time), selected_interval.name)) self.set_header('Content-Type', 'application/json; charset=UTF-8') metrics = list() start_key = None while True: new_metrics = yield gen.Task(metric.Metric.QueryTimespan, self._client, group_key, start_time, end_time, excl_start_key=start_key) if len(new_metrics) > 0: metrics.extend(new_metrics) start_key = metrics[-1].GetKey() else: break data = {'group_key': group_key, 'start_time': start_time, 'end_time': end_time} data['data'] = _SerializeMetrics(metrics) self.write(json.dumps(data)) self.finish()
def getFooter(self): now = time.time() text ="" text+= "<center>" text+= "<table width=100%% border=0 cellspacing=0 cellpadding=0>" text+= " <tr><td colspan=2 height=10><spacer type=block height=10 width=0></td></tr>" text+= " <tr><td colspan=2 bgcolor=#cc0066 height=5><spacer type=block height=5 width=0></td></tr>" text+= " <tr><td colspan=2 height=5><spacer type=block height=5 width=0></td></tr>" text+= " <TR>" text+= " <TD><FONT size=1 >created: %s</font></TD>" % time.ctime(now) #text+= " <TD ALIGN=RIGHT><FONT size=1 >last modified: %s" % time.ctime(now) text+= " <TD ALIGN=RIGHT><FONT size=1 >last modified: %s by" % time.ctime(now) #text+= " <A STYLE=""color: #0000cc"" HREF=""mailto:[email protected]"">[email protected]</A></FONT></TD>" if sys.platform == 'win32': try: user = os.getenv('USERNAME') text+= " <A STYLE=""color: #0000cc"">%s</A></FONT></TD>" % user except: text +="</FONT></TD>" else: try: user = os.getenv("USER") text+= " <A STYLE=""color: #0000cc"">%s</A></FONT></TD>" % user except: text +="</FONT></TD>" text+= " </TR>" text+= "</TABLE>" text+= "</center>" text+= "</BODY>" text+= "</HTML>" return text
def insert_followers_id_2(table_name, target_twitter_id, followers_ids,start_n=0): con = pymysql.connect(host = db_info['db_host'], user = db_info['db_user'], password = db_info['db_password'], db = db_info['db_name']); sqlbase = 'INSERT INTO'+ db_name+'.'+table_name+''' (twitter_Id,follower_Id) VALUES %s;''' sql_values=' ' sql_value_base='('+str(target_twitter_id)+', %s)' #changed something len_ids=len(followers_ids)-1 start_time = time.ctime() i=start_n if i==0: print("Let's do this", start_time) try: #for i in followers_ids: while i <= len_ids: #with con.cursor() as cursor: # insert id one_value=sql_value_base % (followers_ids[i]) #print(one_value) #print(sql) #cursor.execute(sql) #con.commit() if i!=len_ids: if i==0 or i%100!=0: sql_values=sql_values+one_value+',' elif i%100==0 and i!=0: sql_values=sql_values+one_value sql = sqlbase%sql_values #print(sql) with con.cursor() as cursor: cursor.execute(sql) con.commit() now_time=time.ctime() print('%s followers id from %s to %s inserted: %s'%(target_twitter_id, i-100,i,now_time)) sql_values=' ' elif i==len_ids: sql_values=sql_values+one_value sql=sqlbase%sql_values #print(sql) with con.cursor() as cursor: cursor.execute(sql) con.commit() now_time=time.ctime() print('Finished at number: %s'%(i+1)) i+=1 except Exception as e: print("Error %d: %s" % (e.args[0],e.args[1])) print("End at number:", i) con.close() end_time = time.ctime() print('start:',start_time) print('end:', end_time) print('Total ids inserted:',i) return i
def main(): nfuncs = range(len(funcs)) print '*** SINGLE THREAD' for i in nfuncs: print 'starting', funcs[i].__name__, \ 'at:', ctime() print funcs[i](n) print funcs[i].__name__, 'finished at:', \ ctime() print '\n*** MULTIPLE THREADS' threads = [] for i in nfuncs: t = MyThread(funcs[i], (n,), funcs[i].__name__) threads.append(t) for i in nfuncs: threads[i].start() for i in nfuncs: threads[i].join() print threads[i].getResult() print 'all DONE'
def run(self): print 'starting', self.name, 'at:', \ ctime() res = apply(self.func, self.args) self.res = res print self.name, 'finished at:', \ ctime()
def datread(self): """\ Reads in data from .dat or .npy file paths and if .dat is newer than .npy saves it in .npy format for faster reading in future.""" for vals in self.files: if vals['type'] == 'dat' and vals['dir'] == 'pull': # self.datfileloc = vals['dir'] pth = vals['path'] pthnpy = pth.replace('.dat', '.npy', 1) try: print("""\ DAT File:\t{0} DAT: time last modified:\t{2} NPY File:\t{1} NPY: time last modified:\t{3}""".format(pth, pthnpy, \ time.ctime(path.getmtime(pth)), \ time.ctime(path.getmtime(pthnpy)))) if path.getmtime(pth) <= path.getmtime(pthnpy): print('DATFILE: is older than npy... Continue.\n') else: print('DATFILE: is newer than npy... Remaking.\n') os.remove(pthnpy) except OSError: pass if vals['dir'] == 'pull': self.datcheck = True self.path = pth if path.exists(pthnpy): self.dat = np.load(pthnpy) else: self.dat = np.loadtxt(pth) np.save(pthnpy, self.dat)
def up(cursor): to_build = _need_collxml(cursor) num_todo = len(to_build) batch_size = 100 logger.info('collection.xml to generate: {}'.format(num_todo)) logger.info('Batch size: {}'.format(batch_size)) start = time.time() guesstimate = 0.01 * num_todo guess_complete = guesstimate + start logger.info('Completion guess: ' '"{}" ({})'.format(time.ctime(guess_complete), timedelta(0, guesstimate))) num_complete = 0 for batch in _batcher(to_build, batch_size): coll_idents = tuple([i[0] for i in batch]) logger.debug('coll_idents {}'.format(coll_idents)) for coll_ident in coll_idents: _build_collxml(coll_ident, cursor) cursor.connection.commit() num_complete += len(batch) percent_comp = num_complete * 100.0 / num_todo elapsed = time.time() - start remaining_est = elapsed * (num_todo - num_complete) / num_complete est_complete = start + elapsed + remaining_est logger.info('{:.1f}% complete ' 'est: "{}" ({})'.format(percent_comp, time.ctime(est_complete), timedelta(0, remaining_est))) logger.info('Total runtime: {}'.format(timedelta(0, elapsed)))
def set_up_landmines(target): """Does the work of setting, planting, and triggering landmines.""" out_dir = get_target_build_dir(builder(), target, platform() == "ios") landmines_path = os.path.join(out_dir, ".landmines") if not os.path.exists(out_dir): os.makedirs(out_dir) new_landmines = get_landmines(target) if not os.path.exists(landmines_path): with open(landmines_path, "w") as f: f.writelines(new_landmines) else: triggered = os.path.join(out_dir, ".landmines_triggered") with open(landmines_path, "r") as f: old_landmines = f.readlines() if old_landmines != new_landmines: old_date = time.ctime(os.stat(landmines_path).st_ctime) diff = difflib.unified_diff( old_landmines, new_landmines, fromfile="old_landmines", tofile="new_landmines", fromfiledate=old_date, tofiledate=time.ctime(), n=0, ) with open(triggered, "w") as f: f.writelines(diff) elif os.path.exists(triggered): # Remove false triggered landmines. os.remove(triggered)
def MPrun(hostname, scriptfile): ''' Integrate resource allocation information and send it to salt-master :return: send info -> salt-master ''' sys.stdout.write("[ %s ] Start to get the data...\n" % time.ctime()) sys.stdout.flush() global sendinfo sendinfo = {} for f in os.listdir(scriptfile): if os.path.isfile("%s/%s" % (scriptfile, f)): result = commands.getstatusoutput("/usr/bin/python %s/%s" % (scriptfile, f)) if result[0] == 0 and isinstance(eval(result[1]), dict): for k in eval(result[1]).keys(): sendinfo[k] = eval(result[1])[k] else: sys.stdout.write("[ %s ] ERROR : Please check the script for %s and the script return value must be dictionary type\n" % (time.ctime(), f)) sys.stdout.flush() time.sleep(0.5) continue else: sys.stdout.write("[ %s ] ERROR : No such file %s\n" % (time.ctime(), f)) sys.stdout.flush() time.sleep(0.5) continue sys.stdout.write("\n[ %s ] system resource configuration information :\n%s\n\n" % (time.ctime(), sendinfo)) sys.stdout.flush() sendsalt() return
def download_list(srcdirs = [], srcfiles = [], hostname = 'http://127.0.0.1'): dstdirs, dstfiles = visitdir(dstmonitorpath, dstwwwroot) downloadfiles = set(srcfiles) - set(dstfiles) #rmdirs = set(dstdirs) - set(srcdirs) rmfiles = set(dstfiles) - set(srcfiles) logger.warn("rmfiles count=%s", len(rmfiles)) download_count = len(downloadfiles) logger.warn("download tasks count = %s", download_count) expires_time = WHOLE_SYNC_TASK_EXPIRES_TIME + 3600 for f, sz in rmfiles: try: file = join(dstdir, f) localfilesz = None if os.path.exists(file): localfilesz = getsize(file) else: logger.info('file=%s not exist, skip', file) continue last_modified_time = os.path.getmtime(file) time_now = time.time() if time_now - last_modified_time < expires_time: logger.info('file: %s now_time(%s) - last_modified_time(%s) = %ss less than whole_sync_task_expires_time(%ss), skip', file, time.ctime(time_now), time.ctime(last_modified_time), time_now - last_modified_time, expires_time) continue is_skip = False for dlf, dlsz in downloadfiles: if f == dlf: logger.info('file:%s is in download list, suggest_size(%s) =? scan_size(%s), real_size:%s, skip', f, dlsz, sz, localfilesz) is_skip = True break if not is_skip: logger.info('Going to rm file: %s, filesize=%s', file, localfilesz) rmfile(None, file) except Exception as e: logger.exception('check rm file: %s exception', f) if download_count > 0: ThreadPool.initialize() for (f, filesize) in downloadfiles: url = join(hostname, f) ThreadPool.add_task_with_param(download, url) ThreadPool.wait_for_complete(timeout=3600*10) logger.warn("complete downloads") ThreadPool.clear_task() ThreadPool.stop() return 'OK'
def set_up_landmines(target, new_landmines): """Does the work of setting, planting, and triggering landmines.""" out_dir = get_target_build_dir(landmine_utils.builder(), target, landmine_utils.platform() == 'ios') landmines_path = os.path.join(out_dir, '.landmines') if not os.path.exists(out_dir): os.makedirs(out_dir) if not os.path.exists(landmines_path): with open(landmines_path, 'w') as f: f.writelines(new_landmines) else: triggered = os.path.join(out_dir, '.landmines_triggered') with open(landmines_path, 'r') as f: old_landmines = f.readlines() if old_landmines != new_landmines: old_date = time.ctime(os.stat(landmines_path).st_ctime) diff = difflib.unified_diff(old_landmines, new_landmines, fromfile='old_landmines', tofile='new_landmines', fromfiledate=old_date, tofiledate=time.ctime(), n=0) with open(triggered, 'w') as f: f.writelines(diff) elif os.path.exists(triggered): # Remove false triggered landmines. os.remove(triggered)
def github_list_pull_requests(urls, numbers_only=False): """ Returns the pull requests numbers. It returns a tuple of (nonmergeable, mergeable), where "nonmergeable" and "mergeable" are lists of the pull requests numbers. """ pulls = github_get_pull_request_all(urls) formated_pulls = [] print "Total pull count", len(pulls) sys.stdout.write("Processing pulls...") for pull in pulls: n = pull["number"] sys.stdout.write(" %d" % n) sys.stdout.flush() pull_info = github_get_pull_request(urls, n) mergeable = pull_info["mergeable"] if pull["head"]["repo"]: repo = pull["head"]["repo"]["html_url"] else: repo = None branch = pull["head"]["ref"] created_at = pull["created_at"] created_at = time.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ") created_at = time.mktime(created_at) username = pull["head"]["user"]["login"] user_info = github_get_user_info(urls, username) author = "\"%s\" <%s>" % (user_info.get("name", "unknown"), user_info.get("email", "")) formated_pulls.append((created_at, n, repo, branch, author, mergeable)) formated_pulls.sort(key=lambda x: x[0]) print "\nPatches that cannot be merged without conflicts:" nonmergeable = [] for created_at, n, repo, branch, author, mergeable in formated_pulls: if mergeable: continue nonmergeable.append(int(n)) if numbers_only: print n, else: print "#%03d: %s %s" % (n, repo, branch) print unicode(" Author : %s" % author).encode('utf8') print " Date : %s" % time.ctime(created_at) if numbers_only: print print print "-"*80 print "Patches that can be merged without conflicts:" mergeable_list = [] for last_change, n, repo, branch, author, mergeable in formated_pulls: if not mergeable: continue mergeable_list.append(int(n)) if numbers_only: print n, else: print "#%03d: %s %s" % (n, repo, branch) print unicode(" Author : %s" % author).encode('utf8') print " Date : %s" % time.ctime(last_change) if numbers_only: print return nonmergeable, mergeable_list
def run_brute_force(self, keys): """You must set the parameter self.logfile before calling this method. This should probably happen in the __init__ method of a derived class.""" self.bfparams = copy.copy(self.params) self.build_param_mat() self.build_ind_mat() self.bflabels = self.bfkeys+['e','p1','p2'] self.clear_log() start_line = 'start time: ' + time.ctime() print(start_line) self.str_out(start_line+'\n') for row in self.bfinds: self.bfparams = self.set_params(row, self.bfparams) e, th_bode, a_v_bode = self.cost(self.bfparams, extra=1) ## e, th_bode, a_v_bode = _cost(self.bfparams, ## extra=1, \ ## func=self.func, \ ## con_dict=self.con_dict) p1, p2 = self.find_peaks(a_v_bode) self.save_row(self.bfparams, e, p1, p2) print('end time: ' + time.ctime())
async def test_run_task_timeout(context): """`run_task` raises `ScriptWorkerTaskException` and kills the process after exceeding `task_max_timeout`. """ temp_dir = os.path.join(context.config['work_dir'], "timeout") context.config['task_script'] = ( sys.executable, TIMEOUT_SCRIPT, temp_dir ) # With shorter timeouts we hit issues with the script not managing to # create all 6 files context.config['task_max_timeout'] = 5 pre = arrow.utcnow().timestamp with pytest.raises(ScriptWorkerTaskException): await swtask.run_task(context, noop_to_cancellable_process) post = arrow.utcnow().timestamp # I don't love these checks, because timing issues may cause this test # to be flaky. However, I don't want a non- or long- running test to pass. # Did this run at all? assert post - pre >= 5 # Did this run too long? e.g. did it exit on its own rather than killed # If this is set too low (too close to the timeout), it may not be enough # time for kill_proc, kill_pid, and the `finally` block to run assert post - pre < 10 # Did the script generate the expected output? files = {} for path in glob.glob(os.path.join(temp_dir, '*')): files[path] = (time.ctime(os.path.getmtime(path)), os.stat(path).st_size) print("{} {}".format(path, files[path])) for path in glob.glob(os.path.join(temp_dir, '*')): print("Checking {}...".format(path)) assert files[path] == (time.ctime(os.path.getmtime(path)), os.stat(path).st_size) assert len(list(files.keys())) == 6 # Did we clean up? assert context.proc is None
dbc.execute("CREATE TABLE IF NOT EXISTS Pings (url TEXT, time TEXT, response INTEGER)") # # Check if the database is valid # if not isSQLite3(dbFile): sys.stderr.write("The file '{}' is not a valid SQLite database.\n".format(dbFile)) sys.exit(1) # # get current time for logging # rTime = time.ctime(time.time()) try: # url must start with "http://" or the request fails print("Connecting to: {}".format(url)) r = requests.head(url, timeout=5) code = r.status_code except requests.ConnectionError: code = 0 # # insert values into database #
def __init__( self, energy_levels, isotope, levelsfmt, # ='cdsd-pc', use_cached=True, use_json=None, verbose=True, ): # %% Init # Initialize PartitionFunctionCalculator for this electronic state ElecState = ElectronicState("CO2", isotope, "X", "1Σu+") super(PartFuncCO2_CDSDcalc, self).__init__(ElecState) # Check inputs ('return' is not mentionned in signature. it will just return # after cache name is given) assert use_cached in [True, False, "regen", "force", "return"] if isotope not in [1, 2]: raise ValueError( "CDSD Energies not defined for isotope: {0}".format(isotope)) if use_json is not None: warn( DeprecationWarning( "use_json replaced with faster HDF5-based use_cached")) # Get vibrational level definitions that match Energy Database (in particular # how Evib and Erot are calculated) # This is needed to be able to match the levels in the Line Database and # the levels in the Energy database if levelsfmt == "cdsd-p": viblvl_label = "p" elif levelsfmt == "cdsd-pc": viblvl_label = "pc" elif levelsfmt == "cdsd-pcN": viblvl_label = "pcN" elif levelsfmt == "cdsd-hamil": viblvl_label = "pcJN" elif levelsfmt is None: # dont label the levels. Wont be able to use the EnergyDatabase to fetch # vibrational energies for lines, however it can still be used to # calculate Partition functions independently from a Spectrum calculation viblvl_label = None else: raise ValueError( "Unknown Energy database format: levelsfmt = `{0}`".format( levelsfmt) + ". Use one of: `cdsd-p`, `cdsd-pc`, `cdsd-pcN`,`cdsd-hamil`") # Store defaults self.verbose = verbose self.use_cached = use_cached self.levelsfmt = levelsfmt self.viblvl_label = viblvl_label self.last_modification = time.ctime( getmtime(getTestFile(r"co2_cdsd_hamiltonian_fragment.levels"))) if verbose >= 2: print("Last modification time: {0}".format(self.last_modification)) # Get variables to store in metadata (after default values have been set) molecule = "CO2" # will be stored in cache file metadata last_modification = time.ctime( getmtime(getTestFile(r"co2_cdsd_hamiltonian_fragment.levels"))) _discard = [ "self", "energy_levels", "verbose", "ElecState", "electronic_state", "use_json", "use_cached", ] # (dev) locals() automatically stores all variables: levelsfmt, viblvl_label, etc. metadata = filter_metadata(locals(), discard_variables=_discard) # %% Get levels # Function of use_cached value: # ... if True, use (and generate if doesnt exist) cache file. # ... if 'regen', regenerate cache file. If 'force', raise an error # ... if file doesnt exist. # If file is deprecated, regenerate it unless 'force' was used # Load cache file if exists cachefile = energy_levels + ".h5" self.cachefile = cachefile # If return, return after cachefile generated (used for tests) if use_cached == "return": return df = load_h5_cache_file( cachefile, use_cached, metadata=metadata, current_version=radis.__version__, last_compatible_version=OLDEST_COMPATIBLE_VERSION, verbose=verbose, ) if df is None: # Read normal file df = pd.read_csv(energy_levels, comment="#", delim_whitespace=True) df = self._add_degeneracies(df) df = self._add_levels(df) self.df = df # Store if use_cached and not exists(cachefile): save_to_hdf( self.df, cachefile, metadata=metadata, version=radis.__version__, key="df", overwrite=True, verbose=verbose, )
def super_player(file_, time): for i in range(2): print("I was listening to %s !%s" % (file_, ctime())) sleep(3)
t = threading.Thread(target=super_player, args=(file_, time)) threads.append(t) # # 创建线程数组 # threads = [] # # # 创建线程1 # t1 = threading.Thread(target=music, args=('葫芦娃', 2)) # threads.append(t1) # # # 创建线程2 # t2 = threading.Thread(target=movie, args=('蜡笔小新', 2)) # threads.append(t2) if __name__ == '__main__': # music('葫芦娃', 3) # movie("蜡笔小新", 3) # print("over %s" % ctime()) for t in files: # t.start() threads[t].start() # 守护线程 for t in files: # t.join() threads[t].join() print("结束! %s" % ctime())
@property def vertices(self): for vertex in self.bmesh.vertices: yield vertex.co @property def polygons(self): pass @polygons.setter def polygons(self, value): self.polygons.append(Polygon(**value)) if __name__ == '__main__': before = time.ctime(time.time()) object = bpy.context.active_object bmesh = object.data export_mesh = ProcessBMesh(bmesh) # Create a rhom mesh mesh = rhom.Mesh() # Generate the PointTable values for x, y, z in export_mesh.vertices: mesh.addPoint(rhom.Point(x, y, z)) # Write the mesh to disk rhom.writeMesh('Foo.obn', mesh)
def show_time(request): t1 = time.ctime() return render(request, "index.html", {"time1": t1})
''' 题目:时间函数举例1。 ''' if __name__ == '__main__': import time print(time.ctime(time.time())) print(time.asctime(time.localtime(time.time()))) print(time.asctime(time.gmtime(time.time())))
def run(self): logging.info("starting %s at: %s" % (self.name, time.ctime())) self.res = self.func(*self.args) logging.info("finished %s at: %s" % (self.name, time.ctime()))
#Last modified: 11/30/2017 by Jonathan Engelbert # ### No Known Issues ################################################################################################ import arcpy from arcpy import env import sys, string, os, time, datetime # SET TO OVERWRITE arcpy.env.overwriteOutput = True # Logging script myStartDate = str(datetime.date.today()) myStartTime = time.clock() theStartTime = time.ctime() print theStartTime try: myStartDate = str(datetime.date.today()) myStartTime = time.clock() theStartTime = time.ctime() # thisfile = os.path.realpath(__file__) file = open("C:/ETLs/TIM/TIMUpdates/Logs/" + myStartDate + "Transbase2" + ".txt", "w") file.write(theStartTime + "\n") when =datetime.date.today() theDate = when.strftime("%d") theDay=when.strftime("%A") print theDay ################################################################################################
def main(): # pylint: disable=too-many-statements,too-many-branches,too-many-locals """main routine""" if int(platform.python_version().split('.')[0]) < 3: LOGGER.fatal("%s needs at least python version 3, currently %s", ME, platform.python_version()) sys.exit(1) start_time = int(time.time()) _parser = ArgumentParser() _parser.add_argument("-c", "--cfile", dest="configfile", default=ME+".cfg", help="Configuration file", metavar="FILE", required=True) _parser.add_argument("-v", "--verbosity", action="count", default=0, help="increase output verbosity overriding the default") _parser.add_argument("-p", "--parameter", action="store", help="show parameter from configfile") _args = _parser.parse_args() set_logfile(LOGGER, _args.configfile+".log") _config = get_config(_args.configfile, ME) if _args.parameter: if _args.parameter == 'password': print('parameter {}: {}\n'.format(_args.parameter, decrypted(_config[_args.parameter+'_enc']))) else: print('parameter {}: {}\n'.format( _args.parameter, _config[_args.parameter])) sys.exit(0) if _args.verbosity: newLevel = logging.getLogger().getEffectiveLevel() - (_args.verbosity*10) if newLevel < 0: newLevel = 0 LOGGER.warning("Changing loglevel from %d to %d", logging.getLogger().getEffectiveLevel(), newLevel) logging.getLogger().setLevel(newLevel) LOGGER.debug("log level %d", logging.getLogger().getEffectiveLevel()) LOGGER.warning("start python-%s %s-%s pid=%s Connecting ...\n", platform.python_version(), ME, VERSION, os.getpid() ) if _config['password']: LOGGER.warning( "first encrypted the plaintext password and removed from config\n") # we need the password .... _config['password'] = decrypted(_config['password_enc']) # add a few seconds extra to allow the driver timeout handling to do the it's job. # for example, cx_oracle has a cancel routine that we call after a timeout. If # there is a network problem, the cancel gets a ORA-12152: TNS:unable to send break message # setting this defaulttimeout should speed this up socket.setdefaulttimeout(_config['sqltimeout']+3) LOGGER.warning("%s found db_type=%s, driver %s; checking for driver\n", ME, _config['db_type'], _config['db_driver']) if not os.path.exists( os.path.join(_config['checks_dir'], _config['db_type'])): raise ValueError("db_type "+_config['db_type'] + " does not exist in the "+_config['checks_dir']+" directory") db_driver = load_driver(_config) driver_errors = load_driver_errors(_config) db_connections = load_db_connections(_config) LOGGER.info(db_connections) LOGGER.info(driver_errors) LOGGER.info("hostname in zabbix: %s", _config['hostname']) # hide password, hoping username != password ;-) LOGGER.info("connect string : %s\n", db_connections.connect_string(_config).replace(_config['password'], '******')) LOGGER.info('using sql_timeout : %ds\n', _config['sqltimeout']) LOGGER.info("out_file : %s\n", _config['out_file']) if _config['site_checks']: LOGGER.info("site_checks : %s\n", _config['site_checks']) if LOG_CONF: sys_files = 4 else: sys_files = 3 check_files = [{'name': __file__, 'lmod': os.path.getmtime(__file__)}, {'name': db_connections.__file__, 'lmod': os.path.getmtime(db_connections.__file__)}, {'name': driver_errors.__file__, 'lmod': os.path.getmtime(driver_errors.__file__)}, {'name': LOG_CONF, 'lmod': os.path.getmtime(LOG_CONF)} ] if LOG_CONF: check_files.append( {'name': LOG_CONF, 'lmod': os.path.getmtime(LOG_CONF)}) for i in range(sys_files): to_outfile(_config, "{}[checks,{},name]".format(ME, i), check_files[i]['name']) to_outfile(_config, "{}[checks,{},lmod]".format(ME, i), int(check_files[i]['lmod'])) conn_counter = 0 conn_errors = 0 query_counter = 0 query_errors = 0 sleep_c = 0 sleep_s = 1 prev_err = 0 while True: try: for i in range(sys_files): if check_files[i]['lmod'] != os.stat(check_files[i]['name']).st_mtime: LOGGER.warning("%s Changed, from %s to %s restarting ..\n", check_files[i]['name'], time.ctime(check_files[i]['lmod']), time.ctime(os.path.getmtime(check_files[i]['name']))) os.execv(__file__, sys.argv) # reset list in case of a just new connection that reloads the config check_files = [{'name': __file__, 'lmod': os.path.getmtime(__file__)}, {'name': db_connections.__file__, 'lmod': os.path.getmtime(db_connections.__file__)}, {'name': driver_errors.__file__, 'lmod': os.path.getmtime(driver_errors.__file__)}] if LOG_CONF: check_files.append( {'name': LOG_CONF, 'lmod': os.path.getmtime(LOG_CONF)}) _config = get_config(_args.configfile, ME) _config['password'] = decrypted(_config['password_enc']) _start = timer() # hide password, hoping username != password ;-) LOGGER.info('connecting to %s\n', db_connections.connect_string(_config).replace(_config['password'], '******')) conn_has_cancel = False _conn = db_connections.connect(db_driver, _config) if "cancel" in dir(_conn): conn_has_cancel = True LOGGER.info(_conn) conn_counter += 1 to_outfile(_config, ME+"[connect,status]", 0) _cursor = _conn.cursor() connect_info = db_connections.connection_info(_conn) LOGGER.info('connected db_url %s type %s db_role %s version %s\n' '%s user %s %s sid,serial %d,%d instance %s as %s cancel:%s\n', _config['db_url'], connect_info['instance_type'], connect_info['db_role'], connect_info['dbversion'], datetime.datetime.fromtimestamp(time.time()), _config['username'], connect_info['uname'], connect_info['sid'], connect_info['serial'], connect_info['iname'], _config['role'], conn_has_cancel) if connect_info['db_role'] in ["PHYSICAL STANDBY", "SLAVE"]: checks_file = os.path.join(_config['checks_dir'], _config['db_type'], "standby" + "." + connect_info['dbversion'] + ".cfg") else: checks_file = os.path.join(_config['checks_dir'], _config['db_type'], connect_info['db_role'].lower() + "." + connect_info['dbversion']+".cfg") _files = [checks_file] check_files.append({'name': checks_file, 'lmod': 0}) if _config['site_checks']: for addition in _config['site_checks'].split(","): addfile = os.path.join(_config['checks_dir'], _config['db_type'], addition + ".cfg") check_files.append({'name': addfile, 'lmod': 0}) _files.extend([addfile]) LOGGER.info('using checks from %s\n', _files) for checks_file in check_files: if not os.path.exists(checks_file['name']): raise ValueError( "Configfile " + checks_file['name'] + " does not exist") # all checkfiles exist sleep_c = 0 sleep_s = 1 prev_err = 0 con_mins = 0 open_time = int(time.time()) while True: LOGGER.debug("%s while True\n", ME) if connect_info['db_role'] != db_connections.current_role(_conn, connect_info): LOGGER.info("db_role changed from %s to %s", connect_info['db_role'], db_connections.current_role(_conn, connect_info)) # re connect to get the correct monitoring config again break # keep this to compare for when to dump stats now_run = int(time.time()) run_timer = timer() # keep this to compare for when to dump stats # loading checks from the various checkfiles: need_to_load = "no" # pylint: disable=consider-using-enumerate for i in range(len(check_files)): # at 0 - sys_files is the script itself try: current_lmod = os.path.getmtime(check_files[i]['name']) except OSError as _e: LOGGER.warning("%s: %s\n", check_files[i]['name'], _e.strerror) # ignore the error, maybe temporary due to an update current_lmod = check_files[i]['lmod'] if check_files[i]['lmod'] != current_lmod: if i < sys_files: # it is the script, a module or LOG_CONF # that changed LOGGER.warning("%s changed, from %s to %s restarting ...\n", check_files[i]['name'], time.ctime(check_files[i]['lmod']), time.ctime(current_lmod)) os.execv(__file__, sys.argv) else: if check_files[i]['lmod'] == 0: LOGGER.info("checks loading %s\n", check_files[i]['name']) need_to_load = "yes" else: LOGGER.warning("checks changed, reloading %s\n", check_files[i]['name']) need_to_load = "yes" if need_to_load == "yes": to_outfile(_config, ME + "[version]", VERSION) to_outfile( _config, ME + "[config,db_type]", _config['db_type']) to_outfile( _config, ME + "[config,db_driver]", _config['db_driver']) to_outfile( _config, ME + "[config,instance_type]", _config['instance_type']) to_outfile(_config, ME + "[conn,db_role]", connect_info['db_role']) to_outfile( _config, ME + "[conn,instance_type]", connect_info['instance_type']) to_outfile(_config, ME + "[conn,dbversion]", connect_info['dbversion']) to_outfile( _config, ME + "[connect,instance_name]", connect_info['iname']) # sometimes the instance_name query follows within a second # missing event so give it some more time time.sleep(3) objects_list = [] sections_list = [] file_list = [] all_checks = [] for i in range(len(check_files)): _e = collections.OrderedDict() _e = {"{#CHECKS_FILE}": i} file_list.append(_e) files_json = '{\"data\":'+json.dumps(file_list)+'}' to_outfile(_config, ME+".files.lld", files_json) for i in range(sys_files, len(check_files)): # #0 is executable that is also checked for updates # #1 db_connections module # #2 driver_errors module # #3 LOG_CONF if it exists ... # so, skip those and pick the real check_files _checks = configparser.RawConfigParser() try: check_file = open(check_files[i]['name'], 'r') to_outfile(_config, "{}[checks,{},name]".format(ME, i), check_files[i]['name']) to_outfile(_config, "{}[checks,{},lmod]".format(ME, i), str(int(os.stat(check_files[i]['name']).st_mtime))) try: _checks.read_file(check_file) check_file.close() to_outfile(_config, ME + "[checks," + str(i) + ",status]", 0) except configparser.Error: to_outfile(_config, ME + "[checks," + str(i) + ",status]", 13) LOGGER.critical("file %s has parsing errors ->(13)\n", check_files[i]['name']) except IOError as io_error: to_outfile( _config, ME + "[checks," + str(i) + ",status]", 11) LOGGER.critical("file %s IOError %s %s ->(11)\n", check_files[i]['name'], io_error.errno, io_error.strerror) check_files[i]['lmod'] = os.stat( check_files[i]['name']).st_mtime all_checks.append(_checks) for section in sorted(_checks.sections()): sec_mins = int(_checks.get(section, "minutes")) if sec_mins == 0: LOGGER.info( "%s run at connect only\n", section) else: LOGGER.info("%s run every %d minutes\n", section, sec_mins) # dump own discovery items of the queries per section _e = collections.OrderedDict() _e = {"{#SECTION}": section} sections_list.append(_e) _x = dict(_checks.items(section)) for key, sqls in sorted(_x.items()): if sqls and key != "minutes": _d = collections.OrderedDict() _d = {"{#SECTION}": section, "{#KEY}": key} objects_list.append(_d) LOGGER.info("%s: %s\n", key, sqls[0: 60]. replace('\n', ' ').replace('\r', ' ')) # checks are loaded now. sections_json = '{\"data\":'+json.dumps(sections_list)+'}' LOGGER.debug("lld key: %s json: %s\n", ME+".lld", sections_json) to_outfile(_config, ME+".section.lld", sections_json) rows_json = '{\"data\":'+json.dumps(objects_list)+'}' LOGGER.debug("lld key: %s json: %s\n", ME+".lld", rows_json) to_outfile(_config, ME + ".query.lld", rows_json) # sqls can contain multiple statements per key. sqlparse to split them # now. Otherwise use a lot of extra cycles when splitting at runtime # all_sql { {section, key}: statements } all_sql = {} for _checks in all_checks: for section in sorted(_checks.sections()): _x = dict(_checks.items(section)) for key, sqls in sorted(_x.items()): if sqls and key != "minutes": all_sql[(section, key)] = [] for statement in sqlparse.split(sqls): all_sql[(section, key)].append( statement) # checks discovery is also printed # # assume we are still connected. If not, exception will tell real story to_outfile(_config, ME + "[connect,status]", 0) to_outfile(_config, ME + "[uptime]", int(time.time() - start_time)) to_outfile(_config, ME + "[opentime]", int(time.time() - open_time)) # the connect status is only real if executed a query .... for _checks in all_checks: for section in sorted(_checks.sections()): section_timer = timer() # keep this to compare for when to dump stats sec_mins = int(_checks.get(section, "minutes")) if ((con_mins == 0 and sec_mins == 0) or (sec_mins > 0 and con_mins % sec_mins == 0)): # time to run the checks again from this section _x = dict(_checks.items(section)) _cursor = _conn.cursor() for key, sqls in sorted(_x.items()): if sqls and key != "minutes": LOGGER.debug("%s section: %s key: %s\n", ME, section, key) try: query_counter += 1 if conn_has_cancel: # pymysql has no cancel but does have # timeout in connect sqltimeout = threading.Timer( _config['sqltimeout'], cancel_sql, [_conn, section, key]) sqltimeout.start() _start = timer() for statement in all_sql[(section, key)]: LOGGER.debug("%s section: %s key: %s sql: %s\n", ME, section, key, statement) _cursor.execute(statement) startf = timer() # output for the last query must include the # output for the preparing queries is ignored # complete key and value rows = _cursor.fetchall() if conn_has_cancel: sqltimeout.cancel() if "discover" in section: objects_list = [] for row in rows: _d = collections.OrderedDict() for col in range(len(_cursor.description)): _d[_cursor.description[col] [0]] = row[col] objects_list.append(_d) rows_json = '{\"data\":' + \ json.dumps(objects_list)+'}' LOGGER.debug("DEBUG lld key: %s json: %s\n", key, rows_json) to_outfile(_config, key, rows_json) to_outfile(_config, ME + "[query," + section + "," + key + ",status]", 0) else: if rows and len(rows[0]) == 2: _config['section'] = section _config['key'] = key for row in rows: to_outfile( _config, row[0], row[1]) to_outfile(_config, ME + "[query," + section + "," + key + ",status]", 0) elif not rows: to_outfile(_config, ME + "[query," + section + "," + key + ",status]", 0) else: LOGGER.critical('key=%s.%s ZBXDB-%d: ' 'SQL format error: %s\n', section, key, 2, "expect key,value pairs") to_outfile(_config, ME + "[query," + section + "," + key + ",status]", 2) _config['section'] = "" _config['key'] = "" fetchela = timer() - startf elapsed_s = timer() - _start to_outfile(_config, ME + "[query," + section + "," + key + ",ela]", elapsed_s) to_outfile(_config, ME + "[query," + section + "," + key + ",fetch]", fetchela) # except (db_driver.DatabaseError, # socket.timeout) as dberr: except Exception as dberr: if conn_has_cancel: sqltimeout.cancel() ecode, emsg = driver_errors.db_errorcode( db_driver, dberr) elapsed_s = timer() - _start query_errors += 1 to_outfile(_config, ME + "[query," + section + "," + key + ",status]", ecode) to_outfile(_config, ME + "[query," + section + "," + key + ",ela]", elapsed_s) LOGGER.info('key=%s.%s ZBXDB-%s: ' 'Db execution error: %s\n', section, key, ecode, emsg.strip()) if driver_errors.db_error_needs_new_session(db_driver, ecode): raise LOGGER.debug("%s commit\n", ME) _conn.commit() LOGGER.debug("%s committed\n", ME) # end of a section ## time to run the checks again from this section to_outfile(_config, ME + "[query," + section + ",,ela]", timer() - section_timer) # release locks that might have been taken LOGGER.debug("%s commit 2\n", ME) _conn.commit() LOGGER.debug("%s committed.\n", ME) # dump metric for summed elapsed time of this run to_outfile(_config, ME + "[query,,,ela]", timer() - run_timer) to_outfile(_config, ME + "[cpu,user]", resource.getrusage(resource.RUSAGE_SELF).ru_utime) to_outfile(_config, ME + "[cpu,sys]", resource.getrusage(resource.RUSAGE_SELF).ru_stime) to_outfile(_config, ME + "[mem,maxrss]", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) # passed all sections if ((now_run - start_time) % 3600) == 0: gc.collect() # dump stats LOGGER.info("connect %d times, %d fail; started %d queries, " "%d fail memrss:%d user:%f sys:%f\n", conn_counter, conn_errors, query_counter, query_errors, resource.getrusage( resource.RUSAGE_SELF).ru_maxrss, resource.getrusage( resource.RUSAGE_SELF).ru_utime, resource.getrusage(resource.RUSAGE_SELF).ru_stime) # try to keep activities on the same starting second: sleep_time = 60 - ((int(time.time()) - start_time) % 60) LOGGER.debug("Sleeping for %d seconds\n", sleep_time) time.sleep(sleep_time) con_mins = con_mins + 1 # not really mins since the checks could # have taken longer than 1 minute to complete # end while True # except (db_driver.DatabaseError, socket.timeout, ConnectionResetError) as dberr: except Exception as dberr: err_code, err_msg = driver_errors.db_errorcode(db_driver, dberr) elapsed_s = timer() - _start to_outfile(_config, ME + "[connect,status]", err_code) if not driver_errors.db_error_needs_new_session(db_driver, err_code): # from a killed session, crashed instance or similar conn_errors += 1 if prev_err != err_code: sleep_c = 0 sleep_s = 1 prev_err = err_code sleep_c += 1 if sleep_c >= 10: if sleep_s <= 301: # don't sleep longer than 5 mins after connect failures sleep_s += 10 sleep_c = 0 LOGGER.warning('(%d.%d)connection error: [%s] %s for %s@%s\n', sleep_c, sleep_s, err_code, err_msg.strip().replace('\n', ' ').replace('\r', ' '), _config['username'], _config['db_url']) # set_trace() time.sleep(sleep_s) except (KeyboardInterrupt, SystemExit): exit(0)
def __repr__(self): return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i)' \ % (self.nVersion, self.nServices, time.ctime(self.nTime), repr(self.addrTo), repr(self.addrFrom), self.nNonce, self.strSubVer, self.nStartingHeight)
# logging INFO logg.info('default ADDR value implemented') return addr, port def get_msg(msg_in): json_in = json.loads(msg_in) return json_in def send_msg(destination_host, msg): g = json.dumps(msg) destination_host.send(g.encode('utf-8')) # function collecting messages to json def msg_to_json(filename, root_value, message): with open(filename) as f: data = json.load(f) temp = data[root_value] temp.append(message) with open(filename, 'w') as new_f: json.dump(data, new_f, indent=4) timestr = time.ctime(time.time()) + "\n"
def print_progress_info(actual, total): '''print progress info actual/total''' print '# %s %2.2f%% (%s/%s)' % (time.ctime(), total and float((actual * 100)/total) or 0, actual, total)
#coding=utf-8 #创建TCP服务器 from socket import * from time import ctime HOST='' PORT=21567 BUFSIZ=1024 ADDR=(HOST,PORT) tcpSerSock=socket(AF_INET,SOCK_STREAM) #创服务器套接字 tcpSerSock.bind(ADDR) #套接字与地址绑定 tcpSerSock.listen(5) #监听连接,传入连接请求的最大数 while True: print('waiting for connection...') tcpCliSock,addr =tcpSerSock.accept() print('...connected from:',addr) while True: data =tcpCliSock.recv(BUFSIZ).decode() print('date=',data) print("END") if not data: break tcpCliSock.send(('[%s] %s' %(ctime(),data)).encode()) tcpCliSock.close() tcpSerSock.close()
class PythonShellWidget(TracebackLinksMixin, ShellBaseWidget, GetHelpMixin): """Python shell widget""" QT_CLASS = ShellBaseWidget INITHISTORY = ['# -*- coding: utf-8 -*-', '# *** Spyder Python Console History Log ***',] SEPARATOR = '%s##---(%s)---' % (os.linesep*2, time.ctime()) go_to_error = Signal(str) def __init__(self, parent, history_filename, profile=False): ShellBaseWidget.__init__(self, parent, history_filename, profile) TracebackLinksMixin.__init__(self) GetHelpMixin.__init__(self) # Local shortcuts self.shortcuts = self.create_shortcuts() def create_shortcuts(self): array_inline = config_shortcut(lambda: self.enter_array_inline(), context='array_builder', name='enter array inline', parent=self) array_table = config_shortcut(lambda: self.enter_array_table(), context='array_builder', name='enter array table', parent=self) inspectsc = config_shortcut(self.inspect_current_object, context='Console', name='Inspect current object', parent=self) return [inspectsc, array_inline, array_table] def get_shortcut_data(self): """ Returns shortcut data, a list of tuples (shortcut, text, default) shortcut (QShortcut or QAction instance) text (string): action/shortcut description default (string): default key sequence """ return [sc.data for sc in self.shortcuts] #------ Context menu def setup_context_menu(self): """Reimplements ShellBaseWidget method""" ShellBaseWidget.setup_context_menu(self) self.copy_without_prompts_action = create_action(self, _("Copy without prompts"), icon=ima.icon('copywop'), triggered=self.copy_without_prompts) clear_line_action = create_action(self, _("Clear line"), QKeySequence(get_shortcut('console', 'Clear line')), icon=ima.icon('editdelete'), tip=_("Clear line"), triggered=self.clear_line) clear_action = create_action(self, _("Clear shell"), QKeySequence(get_shortcut('console', 'Clear shell')), icon=ima.icon('editclear'), tip=_("Clear shell contents " "('cls' command)"), triggered=self.clear_terminal) add_actions(self.menu, (self.copy_without_prompts_action, clear_line_action, clear_action)) def contextMenuEvent(self, event): """Reimplements ShellBaseWidget method""" state = self.has_selected_text() self.copy_without_prompts_action.setEnabled(state) ShellBaseWidget.contextMenuEvent(self, event) @Slot() def copy_without_prompts(self): """Copy text to clipboard without prompts""" text = self.get_selected_text() lines = text.split(os.linesep) for index, line in enumerate(lines): if line.startswith('>>> ') or line.startswith('... '): lines[index] = line[4:] text = os.linesep.join(lines) QApplication.clipboard().setText(text) #------ Key handlers def postprocess_keyevent(self, event): """Process keypress event""" ShellBaseWidget.postprocess_keyevent(self, event) if QToolTip.isVisible(): _event, _text, key, _ctrl, _shift = restore_keyevent(event) self.hide_tooltip_if_necessary(key) def _key_other(self, text): """1 character key""" if self.is_completion_widget_visible(): self.completion_text += text def _key_backspace(self, cursor_position): """Action for Backspace key""" if self.has_selected_text(): self.check_selection() self.remove_selected_text() elif self.current_prompt_pos == cursor_position: # Avoid deleting prompt return elif self.is_cursor_on_last_line(): self.stdkey_backspace() if self.is_completion_widget_visible(): # Removing only last character because if there was a selection # the completion widget would have been canceled self.completion_text = self.completion_text[:-1] def _key_tab(self): """Action for TAB key""" if self.is_cursor_on_last_line(): empty_line = not self.get_current_line_to_cursor().strip() if empty_line: self.stdkey_tab() else: self.show_code_completion(automatic=False) def _key_ctrl_space(self): """Action for Ctrl+Space""" if not self.is_completion_widget_visible(): self.show_code_completion(automatic=False) def _key_pageup(self): """Action for PageUp key""" pass def _key_pagedown(self): """Action for PageDown key""" pass def _key_escape(self): """Action for ESCAPE key""" if self.is_completion_widget_visible(): self.hide_completion_widget() def _key_question(self, text): """Action for '?'""" if self.get_current_line_to_cursor(): last_obj = self.get_last_obj() if last_obj and not last_obj.isdigit(): self.show_object_info(last_obj) self.insert_text(text) # In case calltip and completion are shown at the same time: if self.is_completion_widget_visible(): self.completion_text += '?' def _key_parenleft(self, text): """Action for '('""" self.hide_completion_widget() if self.get_current_line_to_cursor(): last_obj = self.get_last_obj() if last_obj and not last_obj.isdigit(): self.insert_text(text) self.show_object_info(last_obj, call=True) return self.insert_text(text) def _key_period(self, text): """Action for '.'""" self.insert_text(text) if self.codecompletion_auto: # Enable auto-completion only if last token isn't a float last_obj = self.get_last_obj() if last_obj and not last_obj.isdigit(): self.show_code_completion(automatic=True) #------ Paste def paste(self): """Reimplemented slot to handle multiline paste action""" text = to_text_string(QApplication.clipboard().text()) if len(text.splitlines()) > 1: # Multiline paste if self.new_input_line: self.on_new_line() self.remove_selected_text() # Remove selection, eventually end = self.get_current_line_from_cursor() lines = self.get_current_line_to_cursor() + text + end self.clear_line() self.execute_lines(lines) self.move_cursor(-len(end)) else: # Standard paste ShellBaseWidget.paste(self) #------ Code Completion / Calltips # Methods implemented in child class: # (e.g. InternalShell) def get_dir(self, objtxt): """Return dir(object)""" raise NotImplementedError def get_module_completion(self, objtxt): """Return module completion list associated to object name""" pass def get_globals_keys(self): """Return shell globals() keys""" raise NotImplementedError def get_cdlistdir(self): """Return shell current directory list dir""" raise NotImplementedError def iscallable(self, objtxt): """Is object callable?""" raise NotImplementedError def get_arglist(self, objtxt): """Get func/method argument list""" raise NotImplementedError def get__doc__(self, objtxt): """Get object __doc__""" raise NotImplementedError def get_doc(self, objtxt): """Get object documentation dictionary""" raise NotImplementedError def get_source(self, objtxt): """Get object source""" raise NotImplementedError def is_defined(self, objtxt, force_import=False): """Return True if object is defined""" raise NotImplementedError def show_code_completion(self, automatic): """Display a completion list based on the current line""" # Note: unicode conversion is needed only for ExternalShellBase text = to_text_string(self.get_current_line_to_cursor()) last_obj = self.get_last_obj() if not text: return if text.startswith('import '): obj_list = self.get_module_completion(text) words = text.split(' ') if ',' in words[-1]: words = words[-1].split(',') self.show_completion_list(obj_list, completion_text=words[-1], automatic=automatic) return elif text.startswith('from '): obj_list = self.get_module_completion(text) if obj_list is None: return words = text.split(' ') if '(' in words[-1]: words = words[:-2] + words[-1].split('(') if ',' in words[-1]: words = words[:-2] + words[-1].split(',') self.show_completion_list(obj_list, completion_text=words[-1], automatic=automatic) return obj_dir = self.get_dir(last_obj) if last_obj and obj_dir and text.endswith('.'): self.show_completion_list(obj_dir, automatic=automatic) return # Builtins and globals if not text.endswith('.') and last_obj \ and re.match(r'[a-zA-Z_0-9]*$', last_obj): b_k_g = dir(builtins)+self.get_globals_keys()+keyword.kwlist for objname in b_k_g: if objname.startswith(last_obj) and objname != last_obj: self.show_completion_list(b_k_g, completion_text=last_obj, automatic=automatic) return else: return # Looking for an incomplete completion if last_obj is None: last_obj = text dot_pos = last_obj.rfind('.') if dot_pos != -1: if dot_pos == len(last_obj)-1: completion_text = "" else: completion_text = last_obj[dot_pos+1:] last_obj = last_obj[:dot_pos] completions = self.get_dir(last_obj) if completions is not None: self.show_completion_list(completions, completion_text=completion_text, automatic=automatic) return # Looking for ' or ": filename completion q_pos = max([text.rfind("'"), text.rfind('"')]) if q_pos != -1: completions = self.get_cdlistdir() if completions: self.show_completion_list(completions, completion_text=text[q_pos+1:], automatic=automatic) return #------ Drag'n Drop def drop_pathlist(self, pathlist): """Drop path list""" if pathlist: files = ["r'%s'" % path for path in pathlist] if len(files) == 1: text = files[0] else: text = "[" + ", ".join(files) + "]" if self.new_input_line: self.on_new_line() self.insert_text(text) self.setFocus()
def __repr__(self): return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \ % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
#----------- import time ticks = time.time() #1970纪元后经过的浮点秒数 print ("当前时间戳为:", ticks) localtime = time.localtime(time.time()) print ("本地时间为 :", localtime) print (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) # 将格式字符串转换为时间戳 a = "Sat Mar 28 22:24:24 2016" print (time.mktime(time.strptime(a,"%a %b %d %H:%M:%S %Y"))) import calendar cal = calendar.month(2016, 1) #星期一是默认的每周第一天 print ("以下输出2016年1月份的日历:") print (cal) print("time.ctime() : %s" % time.ctime()) # time.ctime([secs]) 作用相当于asctime(localtime(secs)),未给参数相当于asctime()
class TerminalWidget(ShellBaseWidget): """ Terminal widget """ COM = 'rem' if os.name == 'nt' else '#' INITHISTORY = ['%s *** Spyder Terminal History Log ***' % COM, COM,] SEPARATOR = '%s%s ---(%s)---' % (os.linesep*2, COM, time.ctime()) go_to_error = Signal(str) def __init__(self, parent, history_filename, profile=False): ShellBaseWidget.__init__(self, parent, history_filename, profile) #------ Key handlers def _key_other(self, text): """1 character key""" pass def _key_backspace(self, cursor_position): """Action for Backspace key""" if self.has_selected_text(): self.check_selection() self.remove_selected_text() elif self.current_prompt_pos == cursor_position: # Avoid deleting prompt return elif self.is_cursor_on_last_line(): self.stdkey_backspace() def _key_tab(self): """Action for TAB key""" if self.is_cursor_on_last_line(): self.stdkey_tab() def _key_ctrl_space(self): """Action for Ctrl+Space""" pass def _key_escape(self): """Action for ESCAPE key""" self.clear_line() def _key_question(self, text): """Action for '?'""" self.insert_text(text) def _key_parenleft(self, text): """Action for '('""" self.insert_text(text) def _key_period(self, text): """Action for '.'""" self.insert_text(text) #------ Drag'n Drop def drop_pathlist(self, pathlist): """Drop path list""" if pathlist: files = ['"%s"' % path for path in pathlist] if len(files) == 1: text = files[0] else: text = " ".join(files) if self.new_input_line: self.on_new_line() self.insert_text(text) self.setFocus()
def train(args): np.random.seed(args.seed) if args.cuda: ctx = mx.gpu(0) else: ctx = mx.cpu(0) # dataloader transform = utils.Compose([utils.Scale(args.image_size), utils.CenterCrop(args.image_size), utils.ToTensor(ctx), ]) train_dataset = data.ImageFolder(args.dataset, transform) train_loader = gluon.data.DataLoader(train_dataset, batch_size=args.batch_size, last_batch='discard') style_loader = utils.StyleLoader(args.style_folder, args.style_size, ctx=ctx) print('len(style_loader):',style_loader.size()) # models vgg = net.Vgg16() utils.init_vgg_params(vgg, 'models', ctx=ctx) style_model = net.Net(ngf=args.ngf) style_model.initialize(init=mx.initializer.MSRAPrelu(), ctx=ctx) if args.resume is not None: print('Resuming, initializing using weight from {}.'.format(args.resume)) style_model.load_parameters(args.resume, ctx=ctx) print('style_model:',style_model) # optimizer and loss trainer = gluon.Trainer(style_model.collect_params(), 'adam', {'learning_rate': args.lr}) mse_loss = gluon.loss.L2Loss() for e in range(args.epochs): agg_content_loss = 0. agg_style_loss = 0. count = 0 for batch_id, (x, _) in enumerate(train_loader): n_batch = len(x) count += n_batch # prepare data style_image = style_loader.get(batch_id) style_v = utils.subtract_imagenet_mean_preprocess_batch(style_image.copy()) style_image = utils.preprocess_batch(style_image) features_style = vgg(style_v) gram_style = [net.gram_matrix(y) for y in features_style] xc = utils.subtract_imagenet_mean_preprocess_batch(x.copy()) f_xc_c = vgg(xc)[1] with autograd.record(): style_model.setTarget(style_image) y = style_model(x) y = utils.subtract_imagenet_mean_batch(y) features_y = vgg(y) content_loss = 2 * args.content_weight * mse_loss(features_y[1], f_xc_c) style_loss = 0. for m in range(len(features_y)): gram_y = net.gram_matrix(features_y[m]) _, C, _ = gram_style[m].shape gram_s = F.expand_dims(gram_style[m], 0).broadcast_to((args.batch_size, 1, C, C)) style_loss = style_loss + 2 * args.style_weight * mse_loss(gram_y, gram_s[:n_batch, :, :]) total_loss = content_loss + style_loss total_loss.backward() trainer.step(args.batch_size) mx.nd.waitall() agg_content_loss += content_loss[0] agg_style_loss += style_loss[0] if (batch_id + 1) % args.log_interval == 0: mesg = "{}\tEpoch {}:\t[{}/{}]\tcontent: {:.3f}\tstyle: {:.3f}\ttotal: {:.3f}".format( time.ctime(), e + 1, count, len(train_dataset), agg_content_loss.asnumpy()[0] / (batch_id + 1), agg_style_loss.asnumpy()[0] / (batch_id + 1), (agg_content_loss + agg_style_loss).asnumpy()[0] / (batch_id + 1) ) print(mesg) if (batch_id + 1) % (4 * args.log_interval) == 0: # save model save_model_filename = "Epoch_" + str(e) + "iters_" + str(count) + "_" + str(time.ctime()).replace(' ', '_') + "_" + str( args.content_weight) + "_" + str(args.style_weight) + ".params" save_model_path = os.path.join(args.save_model_dir, save_model_filename) style_model.save_parameters(save_model_path) print("\nCheckpoint, trained model saved at", save_model_path) # save model save_model_filename = "Final_epoch_" + str(args.epochs) + "_" + str(time.ctime()).replace(' ', '_') + "_" + str( args.content_weight) + "_" + str(args.style_weight) + ".params" save_model_path = os.path.join(args.save_model_dir, save_model_filename) style_model.save_parameters(save_model_path) print("\nDone, trained model saved at", save_model_path)
msg.attach(mime) ''' This file takes the SMTP Sender Email address, Receiver Email address as inputs and sends a mail with a specific message using GMail as the SMTP server. This file is useful to send mails from the commandline or programmatically without opening a browser ''' port = 465 # For SSL smtp_server = "smtp.gmail.com" sender_email = "" # Enter sender's email receiver_email = "" # Enter receiver address password = "" #Enter sender's pass date = time.ctime() message = MIMEMultipart() text = MIMEText("""\ Subject: #Subject here. Current Time/Date: {date} #Message here. """) message.attach(text) #attaches text to the email img_data = open('', 'rb').read() #Enter image filename image = MIMEImage(img_data, name=os.path.basename()) #Enter image filename message.attach(image) #attaches image to the email context = ssl.create_default_context() with smtplib.SMTP_SSL(smtp_server, port) as server: server.login(sender_email, password)
import webbrowser import time count = 0 music_videos = [ "https://www.youtube.com/watch?v=M0Sloitb9Jg", "http://www.youtube.com/watch?v=dQw4w9WgXcQ", "https://www.youtube.com/watch?v=iPXKfGxeHIY", "https://www.youtube.com/watch?v=58Le1Ove61o" ] breaks = len(music_videos) print("This program started on " + time.ctime()) while count < breaks: time.sleep(60 * 60 * 2) webbrowser.open(music_videos.pop()) count = count + 1
def main(): #---------------------- startTime = time.time() random.seed(args.seed) mgiJournalsNames = getMgiJournals(args.mgiJournalsFile) retainedSampleSet = None # the final sample sets leftoverSampleSet = None # ... allJournals = set() # all journal names seen in input sampleObjType = getattr(sampleDataLib, args.sampleObjTypeName) for fn in args.inputFiles: verbose("Reading %s\n" % fn) inputSampleSet = sampleDataLib.ClassifiedRefSampleSet( \ sampleObjType=sampleObjType).read(fn) if not retainedSampleSet: # processing 1st input file sampleObjType = inputSampleSet.getSampleObjType() retainedSampleSet = sampleDataLib.ClassifiedRefSampleSet( \ sampleObjType=sampleObjType) leftoverSampleSet = sampleDataLib.ClassifiedRefSampleSet( \ sampleObjType=sampleObjType) verbose("Sample type: %s\n" % sampleObjType.__name__) else: if sampleObjType != inputSampleSet.getSampleObjType(): sys.stderr.write( \ "Input files have inconsistent sample types: %s & %s\n" % \ (sampleObjType.__name__, inputSampleSet.getSampleObjType().__name__) ) exit(5) allJournals |= inputSampleSet.getJournals() for sample in inputSampleSet.sampleIterator(): if args.onlyMgi and not cleanJournalName(sample.getJournal()) \ in mgiJournalsNames: retain = False else: retain = random.random() < float(args.fraction) if retain: retainedSampleSet.addSample(sample) else: leftoverSampleSet.addSample(sample) ### Write output files retainedSampleSet.write(args.retainedFile) leftoverSampleSet.write(args.leftoverFile) ### Write summary report summary = "\nSummary: " summary += "Retaining random set of articles. Fraction: %5.3f\n" \ % args.fraction summary += time.ctime() + '\n' summary += "Seed: %d\n" % args.seed if args.onlyMgi: summary += "Only samples from MGI monitored journals are eligible\n" else: summary += "Samples from any journal are eligible\n" summary += str(args.inputFiles) + '\n' summary += "Input Totals:\n" totRefs = retainedSampleSet.getNumSamples() + \ leftoverSampleSet.getNumSamples() totPos = retainedSampleSet.getNumPositives() + \ leftoverSampleSet.getNumPositives() totNeg = retainedSampleSet.getNumNegatives() + \ leftoverSampleSet.getNumNegatives() summary += formatSummary(totRefs, totPos, totNeg, len(allJournals)) summary += '\n' summary += "Retained Set Totals:\n" summary += formatSummary(retainedSampleSet.getNumSamples(), retainedSampleSet.getNumPositives(), retainedSampleSet.getNumNegatives(), len(retainedSampleSet.getJournals()), ) summary += "(%5.3f%% of inputs)\n" % \ (100.0 * retainedSampleSet.getNumSamples()/totRefs) summary += '\n' summary += "Leftover Set Totals:\n" summary += formatSummary(leftoverSampleSet.getNumSamples(), leftoverSampleSet.getNumPositives(), leftoverSampleSet.getNumNegatives(), len(leftoverSampleSet.getJournals()), ) sys.stdout.write(summary + '\n') return
def decode(self): nla.decode(self) self['latest handshake'] = ctime(self['tv_sec'])
def main(): run_time = time.ctime().replace(' ', '_')[:-8] directory = 'progress/' + run_time if not os.path.exists(directory): os.makedirs(directory) f = open(directory + '/logs.txt', 'w') global args, best_prec1, best_prec1_ps print ("GPU processing available : ", torch.cuda.is_available()) print ("Number of GPU units available :", torch.cuda.device_count()) args = parser.parse_args() ## READ DATA #cudnn.benchmark = True #TODO # Data loading code #traindir = os.path.join(args.data, 'train') #valdir = os.path.join(args.data, 'val') filter_h = [4,6,8] #[1, 3, 5] train_sampler = None train_dataset = TwitterDataset( csv_file='DATA/txt/bamman_clean.txt', folds_file='DATA/folds/fold_0.csv', word_embedding_file='DATA/embeddings/filtered_embs.txt', user_embedding_file='DATA/embeddings/usr2vec.txt', set_type='train', pad = max(filter_h) - 1 ) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True) val_dataset = TwitterDataset( csv_file='DATA/txt/bamman_clean.txt', folds_file='DATA/folds/fold_0.csv', word_embedding_file='DATA/embeddings/filtered_embs.txt', user_embedding_file='DATA/embeddings/usr2vec.txt', set_type='val', pad = max(filter_h) - 1, w2v = train_dataset.w2v ) val_loader = torch.utils.data.DataLoader( val_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True) test_dataset = TwitterDataset( csv_file='DATA/txt/bamman_clean.txt', folds_file='DATA/folds/fold_0.csv', word_embedding_file='DATA/embeddings/filtered_embs.txt', user_embedding_file='DATA/embeddings/usr2vec.txt', set_type='test', pad = max(filter_h) - 1, w2v = train_dataset.w2v ) test_loader = torch.utils.data.DataLoader( test_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True) # epochs # print final acc with config # print best till date for lr in [0.000001, 0.00001]: for wd in [1, 0.1, 0.01, 0.001]: for oc in [50, 300]: for hu in [128, 256]: for dp in [0, 0.1, 0.3]: for fs in [(1,2,3), (1,3,5)]: best_prec1 = 0 best_prec1_index = -1 parameters = {"filters": fs, "out_channels": oc, "max_length": train_dataset.max_l, "hidden_units": hu, "drop_prob": dp, "user_size": 400, "epochs":args.epochs} #device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') model = CUE_CNN(parameters['filters'], parameters['out_channels'], parameters['max_length'], parameters['hidden_units'], parameters['drop_prob'], parameters['user_size'])#.to(device) model = torch.nn.DataParallel(model).cuda() # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.Adam(model.parameters(), lr = lr, weight_decay=wd) # optimizer = torch.optim.SGD(model.parameters(), lr = args.lr, # momentum=args.momentum, # weight_decay=args.weight_decay) # torch.optim.Adadelta(model.parameters(), # rho=args.momentum, # weight_decay=args.weight_decay) # optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.5, patience=10, threshold=0.005) # optionally resume from a checkpoint train_prec1_plot = [] train_loss_plot = [] val_prec1_plot = [] val_loss_plot = [] test_prec1_plot = [] test_loss_plot = [] if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) train_prec1_plot = train_prec1_plot + checkpoint['train_prec1_plot'] train_loss_plot = train_loss_plot + checkpoint['train_loss_plot'] val_prec1_plot = val_prec1_plot + checkpoint['val_prec1_plot'] val_loss_plot = val_loss_plot + checkpoint['val_loss_plot'] test_prec1_plot = test_prec1_plot + checkpoint['test_prec1_plot'] test_loss_plot = test_loss_plot + checkpoint['test_loss_plot'] print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) if args.evaluate: validate(test_loader, model, criterion, f) return for epoch in range(args.start_epoch, args.epochs + args.start_epoch): #TODO #adjust_learning_rate(optimizer, epoch) # train for one epoch train_prec1, train_loss = train(train_loader, model, criterion, optimizer, epoch, f) train_prec1_plot.append(train_prec1) train_loss_plot.append(train_loss) # evaluate on validation set val_prec1, val_loss = validate(val_loader, model, criterion, f) val_prec1_plot.append(val_prec1) val_loss_plot.append(val_loss) scheduler.step(val_loss) # evaluate on test set test_prec1,test_loss = validate(test_loader, model, criterion, f, is_val=False) test_prec1_plot.append(test_prec1) test_loss_plot.append(test_loss) # remember best prec@1 and save checkpoint is_best = val_prec1 > best_prec1 best_prec1 = max(val_prec1, best_prec1) best_prec1_index = epoch if is_best else best_prec1_index save_checkpoint({ 'train_prec1_plot':train_prec1_plot, 'train_loss_plot':train_loss_plot, 'val_prec1_plot':val_prec1_plot, 'val_loss_plot':val_loss_plot, 'test_prec1_plot':test_prec1_plot, 'test_loss_plot':test_loss_plot, 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer' : optimizer.state_dict(), }, is_best) #plot data #plt.figure(figsize=(12,12)) #plt.subplot(2,1,1) #plot_stats(epoch+1, train_loss_plot, val_loss_plot, test_loss_plot, 'train_loss', 'val_loss', 'test_loss', plt) #plt.subplot(2,1,2) #plot_stats(epoch+1, train_prec1_plot, val_prec1_plot, test_prec1_plot, 'train_acc', 'val_acc', 'test_acc', plt) #plt.savefig('progress/' + run_time + '/stats.jpg') #plt.clf() #print "Learning rate is : ", optimizer.param_groups[0]['lr'] print " $$ ", lr, wd, oc, hu, dp, fs, " $$ " print train_prec1_plot[best_prec1_index], best_prec1, test_prec1_plot[best_prec1_index] best_prec1_ps = max(best_prec1, best_prec1_ps) f.write('configuration {0} {1} {2} {3} {4} {5} \n'.format(lr, wd, oc, hu, dp, fs)) f.write('train: {0} val: {1} test: {2} \n'.format(train_prec1_plot[best_prec1_index], best_prec1, test_prec1_plot[best_prec1_index])) f.write('best val performance is : ' + str(best_prec1_ps) + '\n') f.flush() #best_prec1_ps = max(best_prec1, best_prec1_ps) print "final best performance is for val ", best_prec1_ps f.close()
def getXClusters(filename, trainIDs, testIDs, mean_centered, clusterIDs0, clusterIDs1, clusterIDs2): #filename="res/101_trajectories/aug_trajectories-0750am-0805am.txt" path = os.getcwd()+'/' compressed = 'compressed' in filename frameDict = futil.LoadDictFromTxt(path+filename, 'frame') print("Gotten frameDict",time.ctime()) dictOfGrids = futil.GetGridsFromFrameDict(frameDict, mean_centered, compressed) print("Gotten dictOfGrids",time.ctime()) #filepath = makePathMR(filename, '-mergerMinRanges') filepath = makeFullPath(filename, '-mergerRanges.txt') MR = np.loadtxt(filepath, dtype='int') '''MR=MergeRanges. MR[:,0]=merge ids, MR[:,1]=start frame, MR[:,2] = end''' print ("Done loading in getX", time.ctime()) start = getStartVals(filename) Xtrain1 = np.array([]) Xtrain2 = np.array([]) Xtrain0 = np.array([]) #will have numTrain*numFrames rows and size(grid)+1 columns Xtest1 = np.array([]) Xtest2 = np.array([]) Xtest0 = np.array([]) it = 0 trainEmpty = [True]*3 testEmpty = [True]*3 if not numUsing == 0: MR = MR[:numUsing] for row in MR: thisStart = start[it] XVID = sparse.csr_matrix(np.ascontiguousarray(getXInner(row, dictOfGrids,thisStart,frameDict, compressed))) if row[0] in trainIDs: if row[0] in clusterIDs0: if trainEmpty[0] == True: Xtrain0 = XVID trainEmpty[0] = False else: Xtrain0 = sparse.vstack((Xtrain0,XVID))#,axis=0) elif row[0] in clusterIDs1: if trainEmpty[1] == True: Xtrain1 = XVID trainEmpty[1] = False else: Xtrain1 = sparse.vstack((Xtrain1,XVID))#,axis=0) elif row[0] in clusterIDs2: if trainEmpty[2] == True: Xtrain2 = XVID trainEmpty[2] = False else: Xtrain2 = sparse.vstack((Xtrain2,XVID))#,axis=0) print("Finished getting X data for Merger with VID:",row[0]," and it is a training example", time.ctime()) else: if row[0] in clusterIDs0: if testEmpty[0] == True: Xtest0 = XVID testEmpty[0] = False else: Xtest0 = sparse.vstack((Xtest0,XVID))#,axis=0) elif row[0] in clusterIDs1: if testEmpty[1] == True: Xtest1 = XVID testEmpty[1] = False else: Xtest1 = sparse.vstack((Xtest1,XVID))#,axis=0) elif row[0] in clusterIDs2: if testEmpty[2] == True: Xtest2 = XVID testEmpty[2] = False else: Xtest2 = sparse.vstack((Xtest2,XVID))#,axis=0) #Xtest = sparse.vstack((Xtest,XVID))#np.append(Xtest,XVID,axis=0) print("Finished getting X data for Merger with VID:",row[0]," and it is a test example") it += 1 return Xtrain0, Xtrain1, Xtrain2, Xtest0, Xtest1, Xtest2
def makeInactive(self,name): with self.lock: self.active.remove(name) logging.debug('Running {0} at {1}'.format(self.active,time.ctime()))
def run_model(config, seed=0, data_dir='./data', genotype_class='PCDARTS', num_epochs=20, batch_size=get('batch_size'), init_channels=get('init_channels'), train_criterion=torch.nn.CrossEntropyLoss, data_augmentations=None, save_model_str=None, **kwargs): """ Training loop for configurableNet. :param model_config: network config (dict) :param data_dir: dataset path (str) :param num_epochs: (int) :param batch_size: (int) :param learning_rate: model optimizer learning rate (float) :param train_criterion: Which loss to use during training (torch.nn._Loss) :param model_optimizer: Which model optimizer to use during trainnig (torch.optim.Optimizer) :param data_augmentations: List of data augmentations to apply such as rescaling. (list[transformations], transforms.Composition[list[transformations]], None) If none only ToTensor is used :return: """ # instantiate optimize if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) gpu = 'cuda:0' np.random.seed(seed) torch.cuda.set_device(gpu) cudnn.benchmark = True torch.manual_seed(seed) cudnn.enabled=True torch.cuda.manual_seed(seed) logging.info('gpu device = %s' % gpu) logging.info("config = %s", config) if data_augmentations is None: # You can add any preprocessing/data augmentation you want here data_augmentations = transforms.ToTensor() elif isinstance(type(data_augmentations), list): data_augmentations = transforms.Compose(data_augmentations) elif not isinstance(data_augmentations, transforms.Compose): raise NotImplementedError train_dataset = K49(data_dir, True, data_augmentations) test_dataset = K49(data_dir, False, data_augmentations) # train_dataset = KMNIST(data_dir, True, data_augmentations) # test_dataset = KMNIST(data_dir, False, data_augmentations) # Make data batch iterable # Could modify the sampler to not uniformly random sample train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False) genotype = eval("genotypes.%s" % genotype_class) model = Network(init_channels, train_dataset.n_classes, config['n_conv_layers'], genotype) model = model.cuda() total_model_params = np.sum(p.numel() for p in model.parameters()) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = train_criterion criterion = criterion.cuda() if config['optimizer'] == 'sgd': optimizer = torch.optim.SGD(model.parameters(), lr=config['initial_lr'], momentum=config['sgd_momentum'], weight_decay=config['weight_decay'], nesterov=config['nesterov']) else: optimizer = get('opti_dict')[config['optimizer']](model.parameters(), lr=config['initial_lr'], weight_decay=config['weight_decay']) if config['lr_scheduler'] == 'Cosine': lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, num_epochs) elif config['lr_scheduler'] == 'Exponential': lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1) logging.info('Generated Network:') summary(model, (train_dataset.channels, train_dataset.img_rows, train_dataset.img_cols), device='cuda' if torch.cuda.is_available() else 'cpu') for epoch in range(num_epochs): lr_scheduler.step() logging.info('epoch %d lr %e', epoch, lr_scheduler.get_lr()[0]) model.drop_path_prob = config['drop_path_prob'] * epoch / num_epochs train_acc, train_obj = train(train_loader, model, criterion, optimizer, grad_clip=config['grad_clip_value']) logging.info('train_acc %f', train_acc) test_acc, test_obj = infer(test_loader, model, criterion) logging.info('test_acc %f', test_acc) if save_model_str: # Save the model checkpoint, can be restored via "model = torch.load(save_model_str)" if os.path.exists(save_model_str): save_model_str += '_'.join(time.ctime()) torch.save(model.state_dict(), save_model_str) return test_acc
def testStartupHandler(self): # Clean the client records. aff4.FACTORY.Delete(self.client_id, token=self.token) client_mock = action_mocks.ActionMock("SendStartupInfo") for _ in test_lib.TestFlowHelper("ClientActionRunner", client_mock, client_id=self.client_id, action="SendStartupInfo", token=self.token): pass # Check the client's boot time and info. fd = aff4.FACTORY.Open(self.client_id, token=self.token) client_info = fd.Get(fd.Schema.CLIENT_INFO) boot_time = fd.Get(fd.Schema.LAST_BOOT_TIME) self.assertEqual(client_info.client_name, config_lib.CONFIG["Client.name"]) self.assertEqual(client_info.client_description, config_lib.CONFIG["Client.description"]) # Check that the boot time is accurate. self.assertAlmostEqual(psutil.boot_time(), boot_time.AsSecondsFromEpoch()) # Run it again - this should not update any record. for _ in test_lib.TestFlowHelper("ClientActionRunner", client_mock, client_id=self.client_id, action="SendStartupInfo", token=self.token): pass fd = aff4.FACTORY.Open(self.client_id, token=self.token) self.assertEqual(boot_time.age, fd.Get(fd.Schema.LAST_BOOT_TIME).age) self.assertEqual(client_info.age, fd.Get(fd.Schema.CLIENT_INFO).age) # Simulate a reboot in 10 minutes. current_boot_time = psutil.boot_time() psutil.boot_time = lambda: current_boot_time + 600 # Run it again - this should now update the boot time. for _ in test_lib.TestFlowHelper("ClientActionRunner", client_mock, client_id=self.client_id, action="SendStartupInfo", token=self.token): pass # Ensure only this attribute is updated. fd = aff4.FACTORY.Open(self.client_id, token=self.token) self.assertNotEqual(int(boot_time.age), int(fd.Get(fd.Schema.LAST_BOOT_TIME).age)) self.assertEqual(int(client_info.age), int(fd.Get(fd.Schema.CLIENT_INFO).age)) # Now set a new client build time. with test_lib.ConfigOverrider({"Client.build_time": time.ctime()}): # Run it again - this should now update the client info. for _ in test_lib.TestFlowHelper("ClientActionRunner", client_mock, client_id=self.client_id, action="SendStartupInfo", token=self.token): pass # Ensure the client info attribute is updated. fd = aff4.FACTORY.Open(self.client_id, token=self.token) self.assertNotEqual(int(client_info.age), int(fd.Get(fd.Schema.CLIENT_INFO).age))
def create_run_ensemble(model_state_list, n_layers, grad_clip_value=5, seed=0, num_epochs=20, learning_rate=0.001, init_channels=get('init_channels'), batch_size=get('batch_size'), genotype_class='PCDARTS'): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) gpu = 'cuda:0' np.random.seed(seed) torch.cuda.set_device(gpu) cudnn.benchmark = True torch.manual_seed(seed) cudnn.enabled=True torch.cuda.manual_seed(seed) logging.info('gpu device = %s' % gpu) logging.info("config = %s", config) if data_augmentations is None: # You can add any preprocessing/data augmentation you want here data_augmentations = transforms.ToTensor() elif isinstance(type(data_augmentations), list): data_augmentations = transforms.Compose(data_augmentations) elif not isinstance(data_augmentations, transforms.Compose): raise NotImplementedError train_dataset = K49(data_dir, True, data_augmentations) test_dataset = K49(data_dir, False, data_augmentations) # train_dataset = KMNIST(data_dir, True, data_augmentations) # test_dataset = KMNIST(data_dir, False, data_augmentations) # Make data batch iterable # Could modify the sampler to not uniformly random sample train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False) genotype = eval("genotypes.%s" % genotype_class) dataset = dict() dims = [] for i, model_state in enumerate(model_state_list): model = Network(init_channels, train_dataset.n_classes, n_layers, genotype) model.load_state_dict(torch.load(model_state)) model.cuda() for p in model.parameters(): p.requires_grad = False trn_labels = [] trn_features = [] if i == 0: for d,la in train_loader: o = model(Variable(d.cuda())) o = o.view(o.size(0),-1) trn_labels.extend(la) trn_features.extend(o.cpu().data) test_labels = [] test_features = [] for d,la in test_loader: o = model(Variable(d.cuda())) o = o.view(o.size(0),-1) test_labels.extend(la) test_features.extend(o.cpu().data) dataset['trn_labels'] = trn_labels dataset['test_labels'] = test_labels else: for d,la in train_loader: o = model(Variable(d.cuda())) o = o.view(o.size(0),-1) trn_features.extend(o.cpu().data) test_labels = [] test_features = [] for d,la in test_loader: o = model(Variable(d.cuda())) o = o.view(o.size(0),-1) test_features.extend(o.cpu().data) dataset['trn_features'].extend(trn_features) dims.extend(dataset['trn_features'][i][0].size(0)) dataset['test_features'].extend(test_features) trn_feat_dset = FeaturesDataset(dataset['trn_features'][0],dataset['trn_features'][1],dataset['trn_features'][2],dataset['trn_labels']) test_feat_dset = FeaturesDataset(dataset['test_features'][0],dataset['test_features'][1],dataset['test_features'][2],dataset['test_labels']) trn_feat_loader = DataLoader(trn_feat_dset,batch_size=64,shuffle=True) test_feat_loader = DataLoader(val_feat_dset,batch_size=64) model = EnsembleModel(dims, out_size=train_dataset.n_classes) criterion = torch.nn.optim.CrossEntropyLoss criterion = criterion.cuda() optimizer = torch.nn.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9) for epoch in range(num_epochs): epoch_loss, epoch_accuracy = fit(epoch,model,trn_feat_loader,critierion, training=True) val_epoch_loss , val_epoch_accuracy = fit(epoch,model, test_feat_loader, criterion, training=False) if save_model_str: # Save the model checkpoint, can be restored via "model = torch.load(save_model_str)" if not os.path.exists(save_model_str): os.mkdir(save_model_str) torch.save(model.state_dict(), os.path.join(save_model_str, time.ctime()))
params = parse_args() write_args(logger,params) use_numeric,upsampling_rate,target_col = params data = data_reader(data_path,air_path) X,Y = splitXY(data,['delay_time','is_claim','label','cancelled']) x_train,x_test,y_train,y_test = train_test_split( X,Y,test_size=0.1, random_state=SEED ) #upsampling if upsampling_rate>0: indice = upsampling(y_train,upsampling_rate) x_train = x_train.iloc[indice] y_train = y_train.iloc[indice] #feature engineering: logger.writelines('{}: start feature engineering...\n'.format(time.ctime())) x_train = add_features(x_train) x_test = add_features(x_test) # get numeric columns if use_numeric: x_train, x_train_num = numeric_filter(x_train) x_test, x_test_num = numeric_filter(x_test) fitted_dfs,fitted_encoders,encoder_names = categ_encoder( x_train, y_train[target_col],cols=x_train.columns, encoders=('target','count') ) if use_numeric: fitted_dfs.append(x_train_num) train_feat = pd.concat(fitted_dfs,axis=1) train_label = y_train[target_col]