def extract_ipgs(filename, count=None, to_ip=None, from_ip=None):
cache_name = generate_cache_name(filename, '_ipgs', to_ip, from_ip, count)
if os.path.exists(cache_name):
print "Hit a cache extracting IPGs!"
with open(cache_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
return [Decimal(x) for x in (f.readlines()[0].split(','))]
metadatas = extract_expcap_metadatas(filename,
count=count,
to_ip=to_ip,
from_ip=from_ip)
ipgs = []
start = get_first_enabled(metadatas)
if start == None:
return []
last_end = metadatas[start].wire_end_time()
for expcap_packet in metadatas[start + 1:]:
if expcap_packet.is_enabled():
ipgs.append(expcap_packet.wire_end_time() - last_end)
last_end = expcap_packet.wire_end_time()
with open(cache_name, 'w') as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
f.write(','.join([str(ipg) for ipg in ipgs]))
return ipgs
def extract_deltas(filename, column=7, to_ip=None, from_ip=None):
cache_name = generate_cache_name(filename, '_deltas', to_ip, from_ip, None)
if os.path.exists(cache_name):
print "Hit a cache extracting deltas!"
with open(cache_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
return [
Decimal(delta) for delta in f.readlines()[0].split(',')
]
times = extract_times(filename, column, to_ip=to_ip, from_ip=from_ip)
if len(times) == 0:
return []
diffs = []
last_time = times[0]
for pkt_time in times[1:]:
diffs.append(pkt_time - last_time)
last_time = pkt_time
with open(cache_name, 'w') as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
f.write(','.join([str(diff) for diff in diffs]))
return diffs
def train_model(node, freq):
rnn = Sequential()
rnn_layer = GRU(units, input_shape=input_shape)
rnn.add(rnn_layer)
rnn.add(Dropout(0.5))
rnn.add(Dense(1))
opt = adam(lr=0.001, decay=0.0)
rnn.compile(loss='mse', optimizer=opt)
print("COMPILED")
weights_file = weights_dir_base + node + '/' + freq + '.h5'
if exists(weights_file):
with open(weights_file, 'r') as w_file:
with flock.Flock(w_file, flock.LOCK_EX) as lock:
rnn.load_weights(weights_file)
these_epochs = epochs
else:
these_epochs = epochs_init
recent_smooth_lines = subprocess.check_output([
'tail', '-n' + str(train_size + pred_step),
smooth_dir_base + node + "/" + str(freq) + ".out"
])
recent_smooth_lines = recent_smooth_lines.decode("utf-8")
recent_smooths = []
for val in recent_smooth_lines.split("\n")[:-1]:
splits = val.split(",")
if len(splits) == 2:
recent_smooths.append(float(splits[1]))
#recent_smooths = [float(val.split(",")[1]) for val in recent_smooth_lines.split("\n")[:-1]]
input_data = []
output_data = []
print(train_size, pred_step, lb)
for i in range(train_size - lb - pred_step + 1):
input_data.append(recent_smooths[i:i + lb])
output_data.append(recent_smooths[i + lb + pred_step])
input_data = np.reshape(np.array(input_data), (-1, 1, lb))
output_data = np.reshape(np.array(output_data), (-1, 1))
print(input_data.shape)
input_data = normalize(input_data)
output_data = normalize(output_data)
rnn.fit(input_data, output_data, epochs=these_epochs)
out = rnn.predict(input_data)
print(np.min(output_data), np.max(output_data))
print(np.min(out), np.max(out))
if not exists(weights_dir_base + node + '/'):
makedirs(weights_dir_base + node + '/')
with open(weights_file, 'w') as w_file:
with flock.Flock(w_file, flock.LOCK_EX) as lock:
rnn.save_weights(weights_file)
def replace_inet_lib(project_name, git_hash):
directory = INET_LIB_CACHE_DIR + "/" + git_hash
os.makedirs(directory, exist_ok=True)
LOGGER.info("replacing inet lib with the one built from commit " +
git_hash)
with open(directory + "/download.lock", "wb") as lock_file:
LOGGER.info(
"starting download, or waiting for the in-progress download to finish"
)
with flock.Flock(lock_file, flock.LOCK_EX):
LOGGER.info("download lock acquired")
try:
with open(directory + "/libINET.so", "xb") as f:
LOGGER.info(
"we have just created the file, so we need to download it"
)
client = pymongo.MongoClient(MONGO_HOST)
gfs = gridfs.GridFS(client.opp)
LOGGER.info("connected, downloading")
f.write(gfs.get(git_hash).read())
LOGGER.info("download done")
except FileExistsError:
LOGGER.info("the file was already downloaded")
shutil.copy(directory + "/libINET.so", get_project_lib_file(project_name))
LOGGER.info("file copied to the right place")
def extract_times(filename, column=7, count=None, to_ip=None, from_ip=None):
times = []
cache_name = generate_cache_name(filename, '_times', to_ip, from_ip, count)
if os.path.exists(cache_name):
print "Hit a cache extracting arrival times!"
with open(cache_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
return [Decimal(x) for x in (f.readlines()[0].split(','))]
expcap_metadatas = \
extract_expcap_metadatas(filename, count=count, to_ip=to_ip, from_ip=from_ip)
start = get_first_enabled(expcap_metadatas)
for expcap in expcap_metadatas:
if expcap.is_enabled():
times.append(expcap.wire_start_time())
with open(cache_name, 'w') as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
f.write(','.join([str(time) for time in times]))
return times
def extract_sizes(filename, count=None, to_ip=None, from_ip=None):
cache_name = generate_cache_name(filename, '_sizes_list', to_ip, from_ip,
count)
if os.path.exists(cache_name):
print "Hit a cache extracting packet sizes!"
with open(cache_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
sizes = f.readlines()[0].split(',')
return [int(size) for size in sizes]
metadatas = extract_expcap_metadatas(filename,
count,
to_ip=to_ip,
from_ip=from_ip)
sizes = []
for expcap_packet in metadatas:
if expcap_packet.is_enabled():
sizes.append(expcap_packet.length)
with open(cache_name, 'w') as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
f.write(','.join([str(size) for size in sizes]))
return sizes
def main():
func_name = sys._getframe().f_code.co_name
# Load config
config_ini = configparser.ConfigParser()
config_ini.read('conf/config.ini', encoding='utf-8')
os.environ['HTTP_PROXY'] = 'https://proxygate2.nic.nec.co.jp:8080'
os.environ['HTTPS_PROXY'] = 'https://proxygate2.nic.nec.co.jp:8080'
os.environ['NLS_LANG'] = 'Japanese_Japan.AL32UTF8'
# Set logger
log_ini = configparser.ConfigParser()
log_ini.read('conf/log.ini', encoding='utf-8')
log = log_ini['sync_hr']
logfile = config_ini['logger']['logfile']
rotation = config_ini['logger']['rotation']
retention = int(config_ini['logger']['retention'])
logger.remove()
logger.add(logfile, rotation=rotation, retention=retention)
# Set flock object
flock_obj = flock.Flock(config_ini, log_ini)
# Get nowtime
nowtime = datetime.datetime.now()
# Exclusive lock
if not flock_obj.lock():
logger_obj.warning(log['300000'], func_name)
sys.exit()
# Synchronized human resources
hr_obj = hr.Hr(config_ini, log_ini)
# Get nowtime
nowtime = datetime.datetime.now()
# Update employees
hr_obj.update_employees(nowtime)
# Update organizations
hr_obj.update_organizations(nowtime)
# Unlock
flock_obj.unlock()
# Exit main
sys.exit()
def check_msg_queue():
''' 检查环信消息发送队列，发出待发送的消息。'''
with open('/tmp/yym_check_msg_queue.lock', 'w') as f:
blocking_lock = flock.Flock(f, flock.LOCK_EX | flock.LOCK_NB)
try:
with blocking_lock:
print 'Got lock and checking messages queue:'
announce_id_groups = db.session.query(
Message.announce_id, Message.sender_user_id,
Message.content,
Message.ext).filter(Message.pushed == False).group_by(
Message.announce_id, Message.sender_user_id,
Message.content, Message.ext).order_by(
Message.announce_id.desc()).all()
for announce_id_group in announce_id_groups:
announce_id, sender_user_id, content, ext = announce_id_group
query = db.session.query(Message).filter(
Message.pushed == False
).filter(Message.announce_id == announce_id).filter(
Message.sender_user_id == sender_user_id).filter(
Message.content == content).filter(
Message.ext == ext).order_by(Message.id.desc())
messages_groups = group(query, 20)
for messages_group in messages_groups:
messages = messages_group
message = messages[0]
sender = util.get_info_user(message.sender_user_id)
receivers = util.get_info_users(
map(lambda message: message.receiver_user_id,
messages))
if message.announce_id:
announce = util.get_info_announce(
message.announce_id)
msg = u'' if not announce else announce.content
else:
msg = message.content
ext = json.loads(message.ext)
resp = send_message(sender, receivers, msg, ext)
# 记录发送状态
success, result = resp
if success:
for message in messages:
message.pushed = True
db.session.commit()
print '* Sent messages:', ' '.join(
map(lambda x: str(x.id), messages))
except IOError, e:
print 'Checking messages queue job has been under processing!'
def __init__(self, address=0x48, ic=IC_ADS1115, debug=False):
self.dev = I2CDevice(address, debug=debug)
self.address = address
self.debug = debug
self.lock = flock.Flock('/tmp/ADS1X15.lock')
# Make sure the IC specified is valid
if (ic != IC_ADS1015) and (ic != IC_ADS1115):
if self.debug:
print "ADS1x15: Invalid IC specfied: 0x%02X" % ic
else:
self.ic = ic
# Set pga value, so that getLastConversionResult() can use it,
# any function that accepts a pga value must update this.
self.pga = 6144
def record_fixture_fly_towards_the_middle(config_file, fixture_file):
# Initial positions
boids = fl.Flock(config_file)
positions = boids.get_positions().tolist()
velocities = boids.get_velocities().tolist()
# Call method
boids.fly_towards_the_middle()
# Updated positions
positions_after = boids.get_positions().tolist()
velocities_after = boids.get_velocities().tolist()
positions.extend(velocities)
positions_after.extend(velocities_after)
fixture = {"before": positions, "after": positions_after}
fixture_file_handle = open(fixture_file, 'w')
fixture_file_handle.write(yaml.dump(fixture))
fixture_file_handle.close()
def submit():
global name
used = 0
name = byText.formatName(name)
title = titleBoxText.get()
desc = descriptionBoxText.get()
introQuestion = introBoxText.get()
frame.destroy()
print("**************************************")
print("Starting Video Creation")
print("**************************************\n")
print("Getting Lock File...",end="",flush=True)
with open(fileLock_file,'w') as fp:
with flock.Flock(fp,flock.LOCK_EX) as lock:
print("Lock secured")
byText.clearFolders()
byText.makeTitle(introQuestion,introQuestion+"...Be sure to Subscribe!",name)
byText.makeStatic("Next Comment")
for index,i in enumerate(posts):
if i.selected.get()==True and i.parent==0:
list_posts = [i]+i.kids
imgNumber = 0
for post in list_posts:
imgNumber = byText.outImgAndSound(post.textList,post.user,used,imgNumber)+1
byText.makeFullPostVideo(used,name)
used+=1
byText.makeFullVideo(name)
vid_file = "../../out/{}.mp4".format(name)
if len(sys.argv)==1:
print("Video made but not uploaded. Exiting!")
driver.quit()
sys.exit(0)
ret = upload_video(vid_file,title,desc)
if ret==0:
print("Video created!")
else:
print("Failed to create video")
driver.quit()
sys.exit(0)
def __init__(self, parent=None):
super(NGLWidget, self).__init__(parent)
self.cam = Camera()
self.mouseGlobalTX = Mat4()
self.width = 1024
self.height = 720
self.setWindowTitle('Genetic Boids - Generation: ' + str(0) + 'MiniGen: ' + str(0) + '| ' + str(0) + '/' + str(0))
self.spinXFace = 0
self.spinYFace = 0
self.rotate = False
self.translate = False
self.origX = 0
self.origY = 0
self.origXPos = 0
self.origYPos = 0
self.INCREMENT = 0.01
self.ZOOM = 0.1
self.modelPos = Vec3()
self.flock = flock.Flock()
self.flock.AddFood(7)
self.startTimer(updateframes)
self.start = False
win_height = 600
screen = pygame.display.set_mode((win_width, win_height))
fontObj = pygame.font.SysFont("Courier New", 12)
clock = pygame.time.Clock()
done = False
paused = False
# This is a list of circular "obstacles" (pos_vector, rad)
obstacles = []
for i in range(3):
x = random.randint(0, win_width)
y = random.randint(0, win_height)
obstacles.append([math3d.VectorN(x, y), random.randint(50, 150)])
# Create the flock. Flock-members shouldn't spawn on obstacles (if doing the bonus)
F = flock.Flock((0, 0, win_width, win_height), 20, obstacles)
# The mouse position (or None if the user isn't clicking)
mpos = None
# Game Loop
while not done:
# Update
deltaTime = clock.tick() / 1000.0
if paused:
deltaTime = 0.0 # Everything remains functional, but we don't move anything...
F.update(deltaTime, mpos)
# Input
event = pygame.event.poll()
if event.type == pygame.KEYDOWN and event.key == pygame.K_p:
if attempt>30:
sys.exit(-1)
driver.execute_script("arguments[0].scrollIntoView();",lastEl)
print("Searching for more posts...")
time.sleep(5) #give extra time to load new elements
oldPosts = set(posts)
except:
print("Couldnt scroll last element, exit failure")
sys.exit(-1)
break
print("Pulled all text. Making video now...")
driver.quit()
print("Getting Lock File...",end="",flush=True)
with open(fileLock_file,'w') as fp:
with flock.Flock(fp,flock.LOCK_EX) as lock:
print("Lock secured")
clearFolders()
makeStatic("Next Post")
makeThumbnail(subName,episodeNum)
makeTitle(vid_title,voice_text,name)
for i in range(len(listOfPosts)):
postText = listOfPosts[i][0]
user = listOfPosts[i][1]
outImgAndSound(postText,user,i)
makeFullPostVideo(i,name)
makeFullVideo(name)
vid_file = "../../out/{}.mp4".format(name)
vid_title = "Best of {} Episode {} [The Reddit Experiment]".format(urlKey,episodeNum)
vid_des = "Reddit posts directly from {}, read aloud for your convience and pleasure :) ".format(urlKey)
def extract_flow_sizes(filename):
cache_name = generate_cache_name(filename, '_flow_sizes', None, None, None)
if os.path.exists(cache_name):
print "Hit a cache extracting flow sizes!"
with open(cache_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
lines = f.readlines()
if len(lines) == 0:
return []
data = lines[0]
lengths = [int(x) for x in data.split(',')]
return lengths
# Get all the TCP packets, then look for SYNs and FINs.
metadatas = extract_expcap_metadatas(filename)
flows = {}
flow_count = 0
flow_sizes = []
disabled_count = 0
for packet in metadatas:
if packet.is_disabled():
disabled_count += 1
continue
if packet.is_ip() and packet.is_tcp() and packet.is_tcp_syn():
# print "Starting flow for ", packet.packet_data
identifier = tcp_flow_identifier(packet)
if PROCESS_CSV_DEBUG:
print "Flow ID is ", identifier
# print "Start time is", packet.wire_start_time()
flows[identifier] = packet.tcp_data_length
flow_count += 1
elif packet.is_ip() and packet.is_tcp() and (packet.is_tcp_fin()
or packet.is_tcp_rst()):
identifier = tcp_flow_identifier(packet)
if identifier in flows:
flow_sizes.append(packet.tcp_data_length + flows[identifier])
# Remove that flow, we don't want to count
# everything twice. We'll only delete it on
# the first FIN, but that's OK.
del flows[identifier]
elif PROCESS_CSV_DEBUG:
print "Warning! Found a FIN/RST for a flow we didn't see a SYN for!"
elif packet.is_tcp():
identifier = tcp_flow_identifier(packet)
if identifier in flows:
flows[identifier] += packet.tcp_data_length
elif PROCESS_CSV_DEBUG:
print "Saw a TCP packet for a flow we didn't SYN to!"
if len(flows) > 0:
print "Warning: Saw ", len(
flows), " SYNs for flows that weren't closed"
with open(cache_name, 'w') as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
f.write(','.join([str(length) for length in flow_sizes]))
print "Flow count is ", flow_count
print disabled_count, "Packets disbaled"
return flow_sizes
def extract_flow_lengths(filename):
cache_name = generate_cache_name(filename, '_flow_lengths', None, None,
None)
if os.path.exists(cache_name):
print "Hit a cache extracting flow lengths!"
with open(cache_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
lines = f.readlines()
if len(lines) == 0:
return []
data = lines[0]
if data == '':
return []
lengths = [Decimal(x) for x in data.split(',')]
return lengths
# Get all the TCP packets, then look for SYNs and FINs.
metadatas = extract_expcap_metadatas(filename)
debug = False
flows = {}
deleted_flows = {}
flow_count = 0
flow_lengths = []
for packet in metadatas:
if packet.is_disabled():
continue
if packet.is_ip() and packet.is_tcp() and packet.is_tcp_syn():
identifier = tcp_flow_identifier(packet)
flows[identifier] = packet.wire_start_time()
flow_count += 1
if debug:
print "Adding entry for", identifier
if packet.is_ip() and packet.is_tcp() and (packet.is_tcp_fin()
or packet.is_tcp_rst()):
identifier = tcp_flow_identifier(packet)
if identifier in flows:
flow_lengths.append(packet.wire_end_time() - flows[identifier])
# Remove that flow, we don't want to count
# everything twice. We'll only delete it on
# the first FIN, but that's OK.
del flows[identifier]
if debug:
deleted_flows[identifier] = True
print "Removing entry for ", identifier
elif debug:
print "Warning! Found a FIN/RST for a flow we didn't see a SYN for!"
if debug:
if identifier in deleted_flows:
print "Actually, we did, it's just closed!"
if len(flows) > 0:
print "Warning: Saw ", len(
flows), " SYNs for flows that weren't closed"
with open(cache_name, 'w') as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
f.write(','.join([str(length) for length in flow_lengths]))
print "Flow count is ", flow_count
return flow_lengths
def transfer_actions():
''' 处理合并用户行为的任务，同时只允许一个实例运行，防止重复处理同一个任务。'''
with open('/tmp/yym_task_transfer_actions.lock', 'w') as f:
blocking_lock = flock.Flock(f, flock.LOCK_EX | flock.LOCK_NB)
try:
with blocking_lock:
print 'Got lock and processing transferring actions tasks:'
for task in db.session.query(Task).filter(
Task.processed == False).filter(
Task.type == u'transfer_actions').order_by(
Task.id):
print '* Processing task', task.id, task.type, task.data, ':'
data = json.loads(task.data)
from_user_id = data.get('from', None)
to_user_id = data.get('to', None)
if not from_user_id or not to_user_id:
continue
from_user = db.session.query(User).filter(
User.id == from_user_id).first()
to_user = db.session.query(User).filter(
User.id == to_user_id).first()
if not from_user or not to_user:
continue
# ToDo: 这里的替换有可能出现同一个用户多次 follow 一个账号等情况，不确定这是否会影响用户体验和统计数据。
# Comment.user_id :
db.session.query(Comment).filter(
Comment.user_id == from_user_id).update(
{'user_id': to_user_id})
db.session.commit()
# Comment.at_list:
for comment in db.session.query(Comment).filter(
Comment.at_list.ilike(
'%{}%'.format(from_user_id))):
at_list = util.get_ids_from_str(comment.at_list)
at_list_modified = map(
lambda x: to_user_id
if x == from_user_id else x, at_list)
if at_list != at_list_modified:
comment.at_list = ' '.join(
map(str, at_list_modified))
db.session.commit()
# Review.user_id :
db.session.query(Review).filter(
Review.user_id == from_user_id).update(
{'user_id': to_user_id})
db.session.commit()
util.count_reviews([from_user, to_user], [])
# Review.at_list:
for review in db.session.query(Review).filter(
Review.at_list.ilike('%{}%'.format(from_user_id))):
at_list = util.get_ids_from_str(review.at_list)
at_list_modified = map(
lambda x: to_user_id
if x == from_user_id else x, at_list)
if at_list != at_list_modified:
review.at_list = ' '.join(
map(str, at_list_modified))
db.session.commit()
# fans.user_id & fans.fan_id:
to_user.fans.extend(from_user.fans)
from_user.fans = []
to_user.follows.extend(from_user.follows)
from_user.follows = []
util.count_follow_fans([from_user, to_user],
[from_user, to_user])
# favorites.user_id:
to_user.favorites.extend(from_user.favorites)
from_user.favorites = []
util.count_favorites([from_user, to_user], [])
# likes.user_id:
to_user.likes.extend(from_user.likes)
from_user.likes = []
util.count_likes([from_user, to_user], [])
# share_record.user_id:
to_user.share_records.extend(from_user.share_records)
from_user.share_records = []
util.count_shares([from_user, to_user], [], [], [])
db.session.commit()
# message 表没有做迁移。假定消息都能够被及时发出。
# token 表不迁移，保留原始登陆数据。
task.processed = True
db.session.commit()
except IOError, e:
print 'Transferring actions tasks have been under processing!'
def extract_sizes_by_window(filename,
window_size,
count=None,
to_ip=None,
from_ip=None):
cache_name = generate_cache_name(filename,
'_sizes_by_window',
to_ip,
from_ip,
count,
window_size=window_size)
if os.path.exists(cache_name):
print "Hit a cache extracting sizes by window!"
with open(cache_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
lines = f.readlines()
windows = windows_list_from_string(lines[0])
sizes = lines[1].split(',')
for i in xrange(len(sizes)):
if sizes[i] != '':
this_sizes = sizes[i].split('_')
sizes[i] = [int(x) for x in this_sizes]
else:
sizes[i] = []
return windows, sizes
else:
# Try to see if there are any multiple window size files
# that we can build the right one from.
other_name, other_size = get_caches_we_can_build_from(
filename, '_sizes_by_window', to_ip, from_ip, count, window_size)
if other_name != None:
print "We are able to build the graph using a different cache file!"
combination_factor = (int(window_size) / other_size)
print "The combination factor is ", combination_factor
with open(other_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
bigger_windows = []
bigger_sizes = []
lines = f.readlines()
windows = windows_list_from_string(lines[0])
sizes = lines[1].split(',')
for base_index in range(0,
len(sizes) / combination_factor,
combination_factor):
window_start, _ = windows[base_index]
_, window_end = windows[base_index +
combination_factor - 1]
this_window_sizes = []
for index in range(0, combination_factor):
if sizes[base_index + index] != '':
split_sizes = sizes[base_index +
index].split('_')
sub_window_sizes = [
int(x) for x in split_sizes
]
this_window_sizes += sub_window_sizes
bigger_sizes.append(this_window_sizes)
bigger_windows.append((window_start, window_end))
return bigger_windows, bigger_sizes
(windows, packets) = extract_windows(filename,
window_size,
count=count,
to_ip=to_ip,
from_ip=from_ip)
sizes = []
for packet_window in packets:
window_sizes = []
for (fraction_in_window, packet) in packet_window:
window_sizes.append(packet.length)
sizes.append(window_sizes)
with open(cache_name, 'w') as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
f.write(windows_list_to_string(windows))
f.write(','.join([
'_'.join([str(x) for x in size_list]) for size_list in sizes
]))
return windows, sizes
def find_bursts(filename,
count=None,
ipg_threshold=20000,
packet_threshold=20,
to_ip=None,
from_ip=None):
cache_name = generate_cache_name(filename,
'_bursts',
to_ip,
from_ip,
count,
ipg_threshold=ipg_threshold,
packet_threshold=packet_threshold)
if os.path.exists(cache_name):
print "Found a cache! Going to use it!"
with open(cache_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
bursts = expcap_metadata.double_list_load_expcaps_from(f)
return bursts
metadatas = extract_expcap_metadatas(filename,
count=count,
to_ip=to_ip,
from_ip=from_ip)
start = get_first_enabled(metadatas)
if start == None:
return []
time = metadatas[start].wire_end_time()
last_packet = metadatas[start]
burst_count = 0
current_burst = []
bursts = []
print "IPG Threshold is ", ipg_threshold
print "Packet threshold is", packet_threshold
for packet in metadatas[start + 1:]:
if packet.is_disabled():
continue
next_time = packet.wire_start_time()
if next_time - time < ipg_threshold:
burst_count += 1
# A burst starts with the last packet.
if len(current_burst) == 0:
current_burst.append(last_packet)
current_burst.append(packet)
else:
burst_count = 0
if len(current_burst) > packet_threshold:
bursts.append(current_burst)
current_burst = []
current_burst = []
time = packet.wire_end_time()
last_packet = packet
if len(current_burst) > packet_threshold:
bursts.append(current_burst)
total_bursts = sum([len(burst) for burst in bursts])
print "We have a total of ", total_bursts, "packets in bursts"
with open(cache_name, 'w') as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
expcap_metadata.double_list_save_expcaps_to(f, bursts)
return bursts
#!/usr/bin/env python3
# coding: utf-8
import sys
import time
sys.path.append('../')
import flock
import configparser
conf = configparser.ConfigParser()
conf.read('../conf/config.ini', encoding='utf-8')
log = configparser.ConfigParser()
log.read('../conf/log.ini', encoding='utf-8')
flock_obj = flock.Flock(conf, log)
if not flock_obj.lock():
sys.exit()
time.sleep(5)
# Unlock
#flock_obj.unlock()
def run_model(node, freq):
#print("Running model")
rnn = Sequential()
rnn_layer = GRU(units, input_shape=input_shape)
rnn.add(rnn_layer)
rnn.add(Dropout(0.5))
rnn.add(Dense(1))
opt = adam(lr=0.001, decay=0.0)
rnn.compile(loss='mse', optimizer=opt)
weights_file = weights_dir_base + node + '/' + freq + '.h5'
if exists(weights_file):
with open(weights_file, 'r') as w_file:
with flock.Flock(w_file, flock.LOCK_EX) as lock:
rnn.load_weights(weights_file)
else:
print("No weights yet!")
exit(0)
if not exists(predict_dir_base + node + '/'):
makedirs(predict_dir_base + node + '/')
predict_file = predict_dir_base + node + '/' + freq + '.out'
if exists(predict_file):
predict_latest_time = subprocess.check_output(['tail', '-1', predict_file]).decode('utf-8').split(",")[0]
else:
predict_latest_time = None
smooth_file = smooth_dir_base + node + '/' + freq + '.out'
if predict_latest_time:
start_line = subprocess.check_output(
['grep', '-n', predict_latest_time, smooth_file]).decode('utf-8')
start_line = int(start_line.split(":")[0]) - lb + 1
else:
start_line = 0
smooth_lines = subprocess.check_output(['tail', '-n', '+' + str(start_line), smooth_dir_base + node + "/" + str(freq) + ".out"])
smooth_lines = smooth_lines.decode("utf-8")
smooth_lines = smooth_lines.split("\n")
input_smooth_lines = []
for line in smooth_lines[:-1]:
splits = line.split(",")
if len(splits) == 2:
try:
input_smooth_lines.append(float(splits[1]))
except:
pass
#input_smooth_lines = [float(val.split(",")[1]) for val in smooth_lines[:-1]]
print("smooth lines len", len(input_smooth_lines))
timestamps = [val.split(",")[0] for val in smooth_lines[lb - 1:-1]]
#recent_smooths = [float(val.split(",")[1]) for val in recent_smooth_lines.split("\n")[:-1]]
input_data = []
#output_data = []
for i in range(len(input_smooth_lines) - lb + 1):
input_data.append(input_smooth_lines[i:i+lb])
#output_data.append(recent_smooths[i+lb+predict_step])
input_data = np.reshape(np.array(input_data), (-1,1, lb))
input_data = normalize(input_data)
#output_data = np.reshape(np.array(output_data), (-1,1))
results = rnn.predict(input_data)
results = denormalize(results)
print(results)
results = np.reshape(results, (-1))
print(results.shape[0])
print(len(timestamps))
#assert results.shape[0] == len(timestamps)
with open(predict_file, 'a+') as f:
for i in range(results.shape[0]):
f.write(timestamps[i] + "," + str(results[i]) + '\n')
def extract_utilizations(filename,
window_size,
count=None,
to_ip=None,
from_ip=None):
cache_name = generate_cache_name(filename,
'_bandwidths',
to_ip,
from_ip,
count,
window_size=str(window_size))
if os.path.exists(cache_name):
print "Hit a cache extracting utilizations!"
with open(cache_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
lines = f.readlines()
windows = windows_list_from_string(lines[0])
usages = [Decimal(x) for x in (lines[1].split(','))]
return (windows, usages)
else:
# Try to see if there are any multiple window size files
# that we can build the right one from.
other_name, other_size = get_caches_we_can_build_from(
filename, '_bandwidths', to_ip, from_ip, count, window_size)
if other_name != None:
print "We are able to build the graph using a different cache file!"
combination_factor = (int(window_size) / other_size)
print "The combination factor is ", combination_factor
with open(other_name) as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
bigger_windows = []
bigger_utilizations = []
lines = f.readlines()
windows = windows_list_from_string(lines[0])
utilizations = [Decimal(x) for x in lines[1].split(',')]
for base_index in range(0,
len(windows) - combination_factor,
combination_factor):
window_start, _ = windows[base_index]
_, window_end = windows[base_index +
combination_factor - 1]
# Now, go through and build up the appropriate windows:
new_utilizations = []
for index in range(base_index,
base_index + combination_factor):
# We want to weight the contribution to the
# utilization in the new window by the length of
# the window.
sub_window_start, sub_window_end = windows[index]
new_fraction = (sub_window_end - sub_window_start
) / (window_end - window_start)
new_utilizations.append(utilizations[index] *
new_fraction)
bigger_utilizations.append(sum(new_utilizations))
bigger_windows.append((window_start, window_end))
# Before we return, werite the new utiliations out to disk.
save_utilizations_in_cache(cache_name, bigger_windows,
bigger_utilizations)
return bigger_windows, bigger_utilizations
(windows, packets) = extract_windows(filename,
window_size,
count=count,
to_ip=to_ip,
from_ip=from_ip)
usages = []
debug = False
# For each window, go through and sum the total
# fraction of time the window is in use.
for i in xrange(len(windows)):
(window_start, window_end) = windows[i]
total_window_time = window_end - window_start
time_used = Decimal(0.0)
for (fraction, packet) in packets[i]:
if debug:
print fraction
print packet.wire_length_time()
print hash(packet)
time_used += fraction * packet.wire_length_time()
utilization = time_used / total_window_time
usages.append(utilization)
if debug:
print "Window size is ", total_window_time
print "Utilization is ", utilization
print "Number of packets in the window is ", len(packets[i])
if utilization > 1:
print "Greater than one usage: see last ", len(
packets[i]), "utilizations"
save_utilizations_in_cache(cache_name, windows, usages)
return (windows, usages)
def imshow(img,
coords=None,
title='Image',
wait=True,
destroy=True,
save=False,
normalize=False):
global imshow_counter
img = img.copy().astype(np.float32)
def fill_region(dst, y, x, h, w, v):
h2 = h // 2
w2 = w // 2
py = y - h2
px = x - w2
y_min = max(0, py)
y_max = min(dst.shape[0], y + h2)
x_min = max(0, px)
x_max = min(dst.shape[1], x + w2)
if y_max - y_min <= 0 or x_max - x_min <= 0:
return
dst[y_min:y_max, x_min:x_max] = v
if normalize:
img -= np.min(img)
m = np.max(img)
if m != 0.:
img /= m
if save:
if np.all(img <= 1.0):
img *= 255.
if coords is not None:
img = np.copy(img)
if isinstance(coords, pd.DataFrame):
for coord in coords.itertuples():
fill_region(img, int(round(coord.row)) - 2, int(round(coord.col)) - 2, \
5, 5, np.asarray(cld.cls_colors[coord.cls]))
else:
for coord in coords:
fill_region(img, int(round(coord[1])) - 2, int(round(coord[2])) - 2, \
5, 5, np.asarray(cld.cls_colors[coord[0]]))
if save:
if len(img.shape) == 2:
img = img[:, :, None]
lockfile = os.path.join(debug_dir, '.lock')
with open(lockfile, 'w') as fp:
with flock.Flock(fp, flock.LOCK_EX) as lock:
curr_num = len(os.listdir(debug_dir))
filename = os.path.join(debug_dir,
'imshow_%s_%i.png' % (title, curr_num))
cv2.imwrite(filename, img[..., ::-1])
return
plt.title(title)
plt.imshow(img)
plt.show()
if wait:
input('Press enter to continue...')
if destroy:
plt.close()
def save_utilizations_in_cache(cache_name, windows, usages):
with open(cache_name, 'w') as f:
with flock.Flock(f, flock.LOCK_EX) as lock:
f.write(windows_list_to_string(windows))
f.write(','.join([str(usage) for usage in usages]))
