class Types(Evidence):
def load_embedding(self, save_dir):
embed_save_dir = os.path.join(save_dir, 'embed_types')
self.lda = LDA(from_file=os.path.join(embed_save_dir, 'model.pkl'))
def read_data_point(self, program):
types = program['types'] if 'types' in program else []
return list(set(types))
def wrangle(self, data):
return np.array(self.lda.infer(data), dtype=np.float32)
def placeholder(self, config):
return tf.placeholder(tf.float32, [config.batch_size, self.lda.model.n_components])
def encode(self, inputs, config):
with tf.variable_scope('types'):
encoding = tf.layers.dense(inputs, config.units)
return encoding
@staticmethod
def from_call(call):
split = list(reversed([q for q in call.split('(')[0].split('.')[:-1] if q[0].isupper()]))
types = [split[1], split[0]] if len(split) > 1 else [split[0]]
types = [re.sub('<.*', r'', t) for t in types] # ignore generic types in evidence
args = call.split('(')[1].split(')')[0].split(',')
args = [arg.split('.')[-1] for arg in args]
args = [re.sub('<.*', r'', arg) for arg in args] # remove generics
args = [re.sub('\[\]', r'', arg) for arg in args] # remove array type
types_args = [arg for arg in args if not arg == '' and not arg.startswith('Tau_')]
return types + types_args
class APICalls(Evidence):
def load_embedding(self, save_dir):
embed_save_dir = os.path.join(save_dir, 'embed_apicalls')
self.lda = LDA(from_file=os.path.join(embed_save_dir, 'model.pkl'))
def read_data_point(self, program):
apicalls = program['apicalls'] if 'apicalls' in program else []
return list(set(apicalls))
def wrangle(self, data):
return np.array(self.lda.infer(data), dtype=np.float32)
def placeholder(self, config):
return tf.placeholder(tf.float32, [config.batch_size, self.lda.model.n_components])
def encode(self, inputs, config):
with tf.variable_scope('apicalls'):
encoding = tf.layers.dense(inputs, config.units)
return encoding
@staticmethod
def from_call(call):
split = call.split('(')[0].split('.')
cls, name = split[-2:]
return [name] if not cls == name else []
def vocab(clargs):
model = LDA(from_file=clargs.input_file[0])
vocabulary = model.vectorizer.vocabulary_
if clargs.output_file is not None:
with open(clargs.output_file, 'w') as f:
for word in vocabulary:
f.write(word + '\n')
else:
for word in vocabulary:
print(word)
class APICalls(Evidence):
def load_embedding(self, save_dir):
embed_save_dir = os.path.join(save_dir, 'embed_apicalls')
self.lda = LDA(from_file=os.path.join(embed_save_dir, 'model.pkl'))
def read_data_point(self, program):
apicalls = program['apicalls'] if 'apicalls' in program else []
return list(set(apicalls))
def wrangle(self, data):
return np.array(self.lda.infer(data), dtype=np.float32)
def placeholder(self, config):
# type: (object) -> object
return tf.placeholder(tf.float32,
[config.batch_size, self.lda.model.n_components])
def exists(self, inputs):
return tf.not_equal(tf.count_nonzero(inputs, axis=1), 0)
def init_sigma(self, config):
with tf.variable_scope('apicalls'):
self.sigma = tf.get_variable('sigma', [])
def encode(self, inputs, config):
with tf.variable_scope('apicalls'):
encoding = tf.layers.dense(inputs,
self.units,
activation=tf.nn.tanh)
for i in range(self.num_layers - 1):
encoding = tf.layers.dense(encoding,
self.units,
activation=tf.nn.tanh)
w = tf.get_variable('w', [self.units, config.latent_size])
b = tf.get_variable('b', [config.latent_size])
latent_encoding = tf.nn.xw_plus_b(encoding, w, b)
return latent_encoding
def evidence_loss(self, psi, encoding, config):
sigma_sq = tf.square(self.sigma)
loss = 0.5 * (config.latent_size * tf.log(2 * np.pi * sigma_sq + 1e-10)
+ tf.square(encoding - psi) / sigma_sq)
return loss
@staticmethod
def from_call(callnode):
call = callnode['_call']
call = re.sub('^\$.*\$', '', call) # get rid of predicates
split = call.split('(')[0].split('.')
cls, name = split[-2:]
cls = cls.split('<')[
0] # class name might be generic but method name is never
return [name] if not cls == name else []
class Types(Evidence):
def load_embedding(self, save_dir):
embed_save_dir = os.path.join(save_dir, 'embed_types')
self.lda = LDA(from_file=os.path.join(embed_save_dir, 'model.pkl'))
def read_data_point(self, program):
types = program['types'] if 'types' in program else []
return list(set(types))
def wrangle(self, data):
return np.array(self.lda.infer(data), dtype=np.float32)
def placeholder(self, config):
return tf.placeholder(tf.float32, [config.batch_size, self.lda.model.n_components])
def exists(self, inputs):
return tf.not_equal(tf.count_nonzero(inputs, axis=1), 0)
def init_sigma(self, config):
with tf.variable_scope('types'):
self.sigma = tf.get_variable('sigma', [])
def encode(self, inputs, config):
with tf.variable_scope('types'):
encoding = tf.layers.dense(inputs, self.units, activation=tf.nn.tanh)
for i in range(self.num_layers - 1):
encoding = tf.layers.dense(encoding, self.units, activation=tf.nn.tanh)
w = tf.get_variable('w', [self.units, config.latent_size])
b = tf.get_variable('b', [config.latent_size])
latent_encoding = tf.nn.xw_plus_b(encoding, w, b)
return latent_encoding
def evidence_loss(self, psi, encoding, config):
sigma_sq = tf.square(self.sigma)
loss = 0.5 * (config.latent_size * tf.log(2 * np.pi * sigma_sq + 1e-10)
+ tf.square(encoding - psi) / sigma_sq)
return loss
@staticmethod
def from_call(call):
split = list(reversed([q for q in call.split('(')[0].split('.')[:-1] if q[0].isupper()]))
types = [split[1], split[0]] if len(split) > 1 else [split[0]]
types = [re.sub('<.*', r'', t) for t in types] # ignore generic types in evidence
args = call.split('(')[1].split(')')[0].split(',')
args = [arg.split('.')[-1] for arg in args]
args = [re.sub('<.*', r'', arg) for arg in args] # remove generics
args = [re.sub('\[\]', r'', arg) for arg in args] # remove array type
types_args = [arg for arg in args if not arg == '' and not arg.startswith('Tau_')]
return types + types_args
class Keywords(Evidence):
def load_embedding(self, save_dir):
embed_save_dir = os.path.join(save_dir, 'embed_keywords')
self.lda = LDA(from_file=os.path.join(embed_save_dir, 'model.pkl'))
def read_data_point(self, program):
keywords = program['keywords'] if 'keywords' in program else []
return list(set(keywords))
def wrangle(self, data):
return np.array(self.lda.infer(data), dtype=np.float32)
def placeholder(self, config):
return tf.placeholder(tf.float32, [config.batch_size, self.lda.model.n_components])
def exists(self, inputs):
return tf.not_equal(tf.count_nonzero(inputs, axis=1), 0)
def init_sigma(self, config):
with tf.variable_scope('keywords'):
self.sigma = tf.get_variable('sigma', [])
def encode(self, inputs, config):
with tf.variable_scope('keywords'):
encoding = tf.layers.dense(inputs, self.units, activation=tf.nn.tanh)
for i in range(self.num_layers - 1):
encoding = tf.layers.dense(encoding, self.units, activation=tf.nn.tanh)
w = tf.get_variable('w', [self.units, config.latent_size])
b = tf.get_variable('b', [config.latent_size])
latent_encoding = tf.nn.xw_plus_b(encoding, w, b)
return latent_encoding
def evidence_loss(self, psi, encoding, config):
sigma_sq = tf.square(self.sigma)
loss = 0.5 * (config.latent_size * tf.log(2 * np.pi * sigma_sq + 1e-10)
+ tf.square(encoding - psi) / sigma_sq)
return loss
@staticmethod
def split_camel(s):
s = re.sub('(.)([A-Z][a-z]+)', r'\1#\2', s) # UC followed by LC
s = re.sub('([a-z0-9])([A-Z])', r'\1#\2', s) # LC followed by UC
return s.split('#')
@staticmethod
def from_call(call):
qualified = call.split('(')[0]
qualified = re.sub('<.*>', '', qualified).split('.') # remove generics for keywords
kws = list(itertools.chain.from_iterable([Keywords.split_camel(s) for s in qualified]))
return [kw.lower() for kw in kws]
def train(clargs):
print('Reading data file...')
data = get_data(clargs.input_file[0], clargs.evidence)
ok = 'r'
while ok == 'r':
model = LDA(args=clargs)
model.train(data)
top_words = model.top_words(clargs.top)
for i, words in enumerate(top_words):
print('\nTop words in Topic#{:d}'.format(i))
for w in words:
print('{:.2f} {:s}'.format(words[w], w))
if clargs.confirm:
print('\nOK with the model (y(es)/n(o)/r(edo))? ', end='')
ok = sys.stdin.readline().rstrip('\n')
else:
ok = 'y'
if ok == 'y':
print('Saving model to {:s}'.format(
os.path.join(clargs.save, 'model.pkl')))
with open(os.path.join(clargs.save, 'model.pkl'), 'wb') as fmodel:
pickle.dump((model.model, model.vectorizer), fmodel)
class APICalls(Evidence):
def load_embedding(self, save_dir):
embed_save_dir = os.path.join(save_dir, 'embed_apicalls')
self.lda = LDA(from_file=os.path.join(embed_save_dir, 'model.pkl'))
def read_data_point(self, program):
apicalls = program['apicalls'] if 'apicalls' in program else []
return list(set(apicalls))
def wrangle(self, data):
return np.array(self.lda.infer(data), dtype=np.float32)
def placeholder(self, config):
return tf.placeholder(tf.float32, [config.batch_size, self.lda.model.n_components])
def exists(self, inputs):
return tf.not_equal(tf.count_nonzero(inputs, axis=1), 0)
def init_sigma(self, config):
with tf.variable_scope('apicalls'):
self.sigma = tf.get_variable('sigma', [])
def encode(self, inputs, config):
with tf.variable_scope('apicalls'):
encoding = tf.layers.dense(inputs, self.units)
w = tf.get_variable('w', [self.units, config.latent_size])
b = tf.get_variable('b', [config.latent_size])
latent_encoding = tf.nn.xw_plus_b(encoding, w, b)
return latent_encoding
def evidence_loss(self, psi, encoding, config):
sigma_sq = tf.square(self.sigma)
loss = 0.5 * (config.latent_size * tf.log(2 * np.pi * sigma_sq + 1e-10)
+ tf.square(encoding - psi) / sigma_sq)
return loss
@staticmethod
def from_call(call):
split = call.split('(')[0].split('.')
cls, name = split[-2:]
return [name] if not cls == name else []
def load_embedding(self, save_dir):
embed_save_dir = os.path.join(save_dir, 'embed_apicalls')
self.lda = LDA(from_file=os.path.join(embed_save_dir, 'model.pkl'))
class Keywords(Evidence):
STOP_WORDS = { # CoreNLP English stop words
"'ll", "'s", "'m", "a", "about", "above", "after", "again", "against",
"all", "am", "an", "and", "any", "are", "aren't", "as", "at", "be",
"because", "been", "before", "being", "below", "between", "both",
"but", "by", "can", "can't", "cannot", "could", "couldn't", "did",
"didn't", "do", "does", "doesn't", "doing", "don't", "down", "during",
"each", "few", "for", "from", "further", "had", "hadn't", "has",
"hasn't", "have", "haven't", "having", "he", "he'd", "he'll", "he's",
"her", "here", "here's", "hers", "herself", "him", "himself", "his",
"how", "how's", "i", "i'd", "i'll", "i'm", "i've", "if", "in", "into",
"is", "isn't", "it", "it's", "its", "itself", "let's", "me", "more",
"most", "mustn't", "my", "myself", "no", "nor", "not", "of", "off",
"on", "once", "only", "or", "other", "ought", "our", "ours",
"ourselves", "out", "over", "own", "same", "shan't", "she", "she'd",
"she'll", "she's", "should", "shouldn't", "so", "some", "such", "than",
"that", "that's", "the", "their", "theirs", "them", "themselves",
"then", "there", "there's", "these", "they", "they'd", "they'll",
"they're", "they've", "this", "those", "through", "to", "too", "under",
"until", "up", "very", "was", "wasn't", "we", "we'd", "we'll", "we're",
"we've", "were", "weren't", "what", "what's", "when", "when's",
"where", "where's", "which", "while", "who", "who's", "whom", "why",
"why's", "with", "won't", "would", "wouldn't", "you", "you'd",
"you'll", "you're", "you've", "your", "yours", "yourself",
"yourselves", "return", "arent", "cant", "couldnt", "didnt", "doesnt",
"dont", "hadnt", "hasnt", "havent", "hes", "heres", "hows", "im",
"isnt", "its", "lets", "mustnt", "shant", "shes", "shouldnt", "thats",
"theres", "theyll", "theyre", "theyve", "wasnt", "were", "werent",
"whats", "whens", "wheres", "whos", "whys", "wont", "wouldnt", "youd",
"youll", "youre", "youve"
}
def load_embedding(self, save_dir):
embed_save_dir = os.path.join(save_dir, 'embed_keywords')
self.lda = LDA(from_file=os.path.join(embed_save_dir, 'model.pkl'))
def read_data_point(self, program):
keywords = program['keywords'] if 'keywords' in program else []
return list(set(keywords))
def wrangle(self, data):
return np.array(self.lda.infer(data), dtype=np.float32)
def placeholder(self, config):
return tf.placeholder(tf.float32,
[config.batch_size, self.lda.model.n_components])
def exists(self, inputs):
return tf.not_equal(tf.count_nonzero(inputs, axis=1), 0)
def init_sigma(self, config):
with tf.variable_scope('keywords'):
self.sigma = tf.get_variable('sigma', [])
def encode(self, inputs, config):
with tf.variable_scope('keywords'):
encoding = tf.layers.dense(inputs,
self.units,
activation=tf.nn.tanh)
for i in range(self.num_layers - 1):
encoding = tf.layers.dense(encoding,
self.units,
activation=tf.nn.tanh)
w = tf.get_variable('w', [self.units, config.latent_size])
b = tf.get_variable('b', [config.latent_size])
latent_encoding = tf.nn.xw_plus_b(encoding, w, b)
return latent_encoding
def evidence_loss(self, psi, encoding, config):
sigma_sq = tf.square(self.sigma)
loss = 0.5 * (config.latent_size * tf.log(2 * np.pi * sigma_sq + 1e-10)
+ tf.square(encoding - psi) / sigma_sq)
return loss
@staticmethod
def split_camel(s):
s = re.sub('(.)([A-Z][a-z]+)', r'\1#\2', s) # UC followed by LC
s = re.sub('([a-z0-9])([A-Z])', r'\1#\2', s) # LC followed by UC
return s.split('#')
@staticmethod
def from_call(callnode):
call = callnode['_call']
call = re.sub('^\$.*\$', '', call) # get rid of predicates
qualified = call.split('(')[0]
qualified = re.sub('<.*>', '', qualified).split(
'.') # remove generics for keywords
# add qualified names (java, util, xml, etc.), API calls and types
keywords = list(chain.from_iterable([Keywords.split_camel(s) for s in qualified])) + \
list(chain.from_iterable([Keywords.split_camel(c) for c in APICalls.from_call(callnode)])) + \
list(chain.from_iterable([Keywords.split_camel(t) for t in Types.from_call(callnode)]))
# convert to lower case, omit stop words and take the set
return list(
set([
k.lower() for k in keywords
if k.lower() not in Keywords.STOP_WORDS
]))
