def queryDatalog(self, datalog, **kwargs):
if 'loglevel' in kwargs:
set_logger(kwargs['loglevel'], kwargs.get('debug', False))
logger.info("query datalog:\n{}\n".format(pprint.pformat(datalog)))
self.analyze(datalog, **kwargs)
source_results = []
if self.mode in ['single', 'union']:
source_results = [
self.querySubDatalog(p, **kwargs) for p in self.parsers
]
elif self.mode == 'single_view':
view_schema = {}
for p in self.parsers[:-1]:
source = 'postgres' if 'postgres' in p.query_columns else 'view'
view_schema[p.return_table] = [
col['column'] for col in p.query_columns[source]
]
source_result = None
for p in self.parsers[:-1]:
if not source_result:
source_result = self.querySubDatalog(
p, returnview=p.return_table, **kwargs)
else:
source_result = self.querySubDatalog(
p,
withview=source_result,
viewschema=view_schema,
returnview=p.return_table,
**kwargs)
source_results = [
self.querySubDatalog(self.parsers[-1],
withview=source_result,
viewschema=view_schema,
**kwargs)
]
elif self.mode == 'single_union':
None
elif self.mode in ['view']:
source_results = [
self.querySubDatalog(p, **kwargs) for p in self.parsers[:1]
]
if 'debug' in kwargs: return None
combiner = Combiner(self.mode, source_results, self.parsers)
return combiner.process(**kwargs)
def run(name, configFile, ttsConfigFile, contentFile, videoFile, logFile):
OutputPath.init(configFile)
#thread = ThreadWritableObject(configFile, name, logFile)
#thread.start()
#sys.stdout = thread
#sys.errout = thread # XXX: Actually, it does NOT work
try:
print('Now: ', datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
Network.setIsEnabled(True)
tts = Tts(ttsConfigFile)
combiner = Combiner(configFile)
combiner.combine(tts, contentFile, videoFile)
except KeyboardInterrupt:
pass
except Exception as e:
print('Error occurs at', datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
traceback.print_exc(file=sys.stdout)
finally:
pass
def __init__(self, node='root'):
super(Affixer, self).__init__()
self.scanner = BiLSTMScanner(hidden_size=1)
self.pivoter = BiScanner(morpho_size=tpr.dmorph + 2,
nfeature=5,
node=node + '-pivoter')
self.stem_modifier = StemModifier()
if node == 'root':
self.reduplicator = Affixer('reduplicant')
self.unpivoter = BiScanner(morpho_size=tpr.dmorph + 2,
nfeature=5,
node=node + '-unpivoter')
self.redup = Parameter(
torch.zeros(1)) # xxx need to modulate by morph
self.affix_thunker = Thunker()
self.combiner = Combiner()
self.node = node
'''
'''
web_crawler = WebCrawler()
image_analyzer = ImageAnalyzer()
cur_file = os.getcwd()
dm = DataMiner(cur_file, web_crawler, 'http://www.bbc.com/travel?referer=https%3A%2F%2Fwww.bbc.com%2Fnews%2Flive%2Fworld-53039952')
dm.gather_text(show_status = True, mx_phrazes = 40)
dm.gather_images(web_crawler, mx_images = 10)
dm.generate_images_captions(image_analyzer)
'''
text_classifier = TextClassifierDenseBBC()
text_classifier.build_model()
text_classifier.train_model(show_status=True)
combiner = Combiner(os.path.join(os.getcwd(), 'Data', 'TextFiles'),
text_classifier)
combiner.process_text_data()
text_report, image_report = combiner.get_data_report()
'''
combiner = Combiner()
# build and train text classifier
text_classifier = TextClassifierDenseBBC()
text_classifier.build_model()
text_classifier.train_model(show_status = True)
# train phaze is finished
combiner = Combiner()
def __init__(self,
num_embeddings,
embedding_dims,
edge_output_size,
device,
w,
is_att=False,
transfer=False,
nor=0,
if_no_time=0,
threhold=None,
second_order=False,
if_updated=0,
drop_p=0,
num_negative=5,
act='tanh',
if_propagation=1,
decay_method='exp',
weight=None,
relation_size=None,
bias=True):
super(DyGNN, self).__init__()
self.embedding_dims = embedding_dims
self.num_embeddings = num_embeddings
self.nor = nor
#self.weight = weight.to(device)
self.device = device
self.transfer = transfer
self.if_propagation = if_propagation
self.if_no_time = if_no_time
self.second_order = second_order
# self.cuda = cuda
self.combiner = Combiner(embedding_dims, embedding_dims,
act).to(device)
self.decay_method = decay_method
self.if_updated = if_updated
self.threhold = threhold
print('Only propagate to relevance nodes below time interval: ',
threhold)
# self.tanh = nn.Tanh().to(device)
if act == 'tanh':
self.act = nn.Tanh().to(device)
elif act == 'sigmoid':
self.act = nn.Sigmoid().to(device)
else:
self.act = nn.ReLU().to(device)
self.decayer = Decayer(device, w, decay_method)
self.edge_updater_head = Edge_updater_nn(embedding_dims,
edge_output_size, act,
relation_size).to(device)
self.edge_updater_tail = Edge_updater_nn(embedding_dims,
edge_output_size, act,
relation_size).to(device)
if if_no_time:
self.node_updater_head = nn.LSTMCell(edge_output_size,
embedding_dims,
bias).to(device)
self.node_updater_tail = nn.LSTMCell(edge_output_size,
embedding_dims,
bias).to(device)
else:
self.node_updater_head = TLSTM(edge_output_size,
embedding_dims).to(device)
self.node_updater_tail = TLSTM(edge_output_size,
embedding_dims).to(device)
self.tran_head_edge_head = nn.Linear(edge_output_size, embedding_dims,
bias).to(device)
self.tran_head_edge_tail = nn.Linear(edge_output_size, embedding_dims,
bias).to(device)
self.tran_tail_edge_head = nn.Linear(edge_output_size, embedding_dims,
bias).to(device)
self.tran_tail_edge_tail = nn.Linear(edge_output_size, embedding_dims,
bias).to(device)
self.is_att = is_att
if self.is_att:
self.attention = Attention(embedding_dims).to(device)
self.num_negative = num_negative
self.recent_timestamp = torch.zeros((num_embeddings, 1),
dtype=torch.float,
requires_grad=False).to(device)
self.interaction_timestamp = lil_matrix(
(num_embeddings, num_embeddings), dtype=np.float32)
self.cell_head = nn.Embedding(num_embeddings, embedding_dims,
weight).to(device)
self.cell_head.weight.requires_grad = False
self.cell_tail = nn.Embedding(num_embeddings, embedding_dims,
weight).to(device)
self.cell_tail.weight.requires_grad = False
self.hidden_head = nn.Embedding(num_embeddings, embedding_dims,
weight).to(device)
self.hidden_head.weight.requires_grad = False
self.hidden_tail = nn.Embedding(num_embeddings, embedding_dims,
weight).to(device)
self.hidden_tail.weight.requires_grad = False
self.node_representations = nn.Embedding(num_embeddings,
embedding_dims,
weight).to(device)
self.node_representations.weight.requires_grad = False
if transfer:
self.transfer2head = nn.Linear(embedding_dims, embedding_dims,
False).to(device)
self.transfer2tail = nn.Linear(embedding_dims, embedding_dims,
False).to(device)
if drop_p >= 0:
self.dropout = nn.Dropout(p=drop_p).to(device)
self.cell_head_copy = nn.Embedding.from_pretrained(
self.cell_head.weight.clone()).to(device)
self.cell_tail_copy = nn.Embedding.from_pretrained(
self.cell_tail.weight.clone()).to(device)
self.hidden_head_copy = nn.Embedding.from_pretrained(
self.hidden_head.weight.clone()).to(device)
self.hidden_tail_copy = nn.Embedding.from_pretrained(
self.hidden_tail.weight.clone()).to(device)
self.node_representations_copy = nn.Embedding.from_pretrained(
self.node_representations.weight.clone()).to(device)
from combiner import Combiner from analyzer import Analyzer from combiner_config import MessagesConfig, PostsConfig, LocationsConfig, LikesConfig, CommentsConfig from analyzer_config import MessagesAnalyzerConfig, CommentsAnalyzerConfig, PostsAnalyzerConfig combiner = Combiner(config=PostsConfig) files = combiner.combine() analyzer = Analyzer(config=PostsAnalyzerConfig) analyzer.analyze()
import sys
from combiner import Combiner
from config.config_importer import ConfigImporter
from filter import Filter
import csv_exporter
import csv_importer
args = sys.argv
if len(args) != 4:
print("""
Usage:
python main.py <input_file.csv> <output_file.csv> <config_file.json>
""")
exit()
equipment_pieces = csv_importer.import_file(args[1])
config = ConfigImporter(args[3]).load()
combinations = Combiner(equipment_pieces).generate_combinations()
filtered_combinations = Filter(config, combinations).filter()
csv_exporter.export_combinations(equipment_pieces, filtered_combinations,
args[2])
import sys
from base import ctx
from combiner import Combiner
from excepthook import excepthook
if __name__ == "__main__":
# Install global exception hook
sys._excepthook = sys.excepthook
sys.excepthook = excepthook
app = Combiner(ctx)
exit_code = app.run()
sys.excepthook = excepthook
sys.exit(exit_code)