def main():
# Grab file_path argument from initial run
file_path = sys.argv[1]
#Instatiate repl that interacts with the user
repl = REPL()
repl.greet()
# Instantiate ip populator that generates all the ips
ip_populator = IpPopulator(file_path=file_path)
# Save file to json if it doesnt already exist
json_file_path = file_path.replace("txt", "json")
if path.exists(json_file_path) is False:
exporter = Exporter()
exporter.export_as_json(
file_path=json_file_path,
modeled_ips_dict=ip_populator.dict_of_ip_objects)
# Set queryable_options for repl
repl.queryable_options = ip_populator.list_of_ip_geo_attributes
repl.ip_populator = ip_populator
# Get into query mode
response = repl.intro()
repl.handle_response(response)
#iterate over the subsection of ip addresses for testing purposes
# write json file of ips
import pdb
pdb.set_trace()
def main(self):
self.resolveArgs()
cveList = self.load()
result = None
print("Cve(s) cargados: {}".format(len(cveList)))
if (self.args.source == 'circl'):
src = CirclSource()
src.setCVEList(cveList)
result = src.process()
elif (self.args.source == 'rfl' or self.args.source == 'rfr'):
mode = None
if self.args.source == 'rfl':
mode = 1
elif self.args.source == 'rfr':
mode = 2
opts = {'mode': mode}
src = RFSource()
src.setCVEList(cveList)
result = src.process(opts)
if result == None:
print('Error desconocido my bro')
else:
exporter = Exporter()
exporter.export(result, 'out.csv')
def test_export_range(
self,
mocked_upload,
import_test_data,
start_date_str,
end_date_str,
num_expected_files,
total_expected_lines,
):
exporter = Exporter()
start_date = datetime.strptime(start_date_str, "%Y%m%d").date()
end_date = datetime.strptime(end_date_str, "%Y%m%d").date()
exporter.export_range(start_date, end_date)
num_days = (end_date - start_date).days + 1
batch_names = {(start_date + timedelta(days=x)).strftime("%Y%m%d")
for x in range(num_days)}
batch_names = list(batch_names.intersection(BATCHES_IN_TEST_FILE))
total_lines = 0
for batch_name in batch_names:
filename = f"{batch_name}_NPR_BACKUP.csv"
path = os.path.join(settings.TMP_DATA_DIR, filename)
mocked_upload.assert_any_call(path, filename)
assert exists(path)
with open(path) as f:
lines = f.readlines()
total_lines += len(lines) - 1 # don't count the header
# cleanup
os.remove(path)
assert total_lines == total_expected_lines
def main(self):
flat_playlist_settings = self.config.get('flat_playlist_settings')
if not flat_playlist_settings:
flat_playlist_settings = {'playlist_items': '0-100'}
download_list = self.flat_playlist(options=flat_playlist_settings)
downloaded_songs = list()
for song_url in download_list:
resp = self.download_url(song_url, download=True)
if resp[0]:
downloaded_songs.append(resp[1])
for song_dict in downloaded_songs:
#Manual Tags
#song_dict["album"] = "Ultimate Christmas Hits"
artist = song_dict["artist"]
#artist = ""
#Is a cover wanted?
coverwanted = True
#define export dir without title
export_dir = "{0}/{1}/{2}/".format(\
self.pretty_path(self.config["export_dir"]) \
,self.pretty_path(artist) \
,self.pretty_path(song_dict["album"]))
#check if path exists
if not os.path.exists(export_dir):
os.makedirs(export_dir)
#append title
export_path = export_dir + self.pretty_path(song_dict["title"])
#debug
print("Export_Path: " + export_path)
#convert cover
resp_cover = self.download_crop_cover(song_dict["cover"])
if coverwanted and resp_cover[0]:
song_dict.update({"cover": resp_cover[1]})
else:
song_dict.update({"cover": None})
ytdl_tmp_path = song_dict.pop("ytdl_tmp_path")
try:
exporter = Exporter(ytdl_tmp_path, song_dict)
exporterlog = exporter.export(export_path)
except Exception as e:
print("skipped: {0}\nError: {1}".format(ytdl_tmp_path, e))
else:
print("successfully exported: {0}\n{1}".format(
song_dict["title"], exporterlog))
print("Downloaded {0}:songs from url: {1}".format(
len(downloaded_songs), self.url))
def send_doc():
patient_id = request.args.get('pt_id','')
recipient = request.args.get('addr','')
logging.debug("Sending email for patient id "+patient_id)
exporter = Exporter(patient_id)
exporter.collect_data(recipient)
return render_template('success_page.html')
def export_detail(self, order_id):
session = db_session()
order = session.query(Order).filter(Order.order_id == order_id).one()
exporter = Exporter()
output = exporter.export_detail(order.content, order.order_id.__str__() + '_内部版')
session.close()
return output
def main(argv):
ws = WorkSpace(args['infile'])
if args['plot'] == 'log':
ws.plot2d_log()
elif args['plot'] == 'linear':
ws.plot2d()
ws.info()
exporter = Exporter(args['outfile'], ws)
exporter.export()
def initialize(self, json_export_path=None):
print("Initialising PoissonSolver...")
self.metal_params = helpers.getMetalParams(self.sqconn)
self.elec_list = helpers.getElectrodeCollections(self.sqconn)
self.db_list = helpers.getDBCollections(self.sqconn)
self.initParameters(self.sqconn.getAllParameters())
self.sim_params = self.sqconn.getAllParameters()
self.createBoundaries()
self.exporter = Exporter(in_path=self.sqconn.inputPath(),
out_path=self.sqconn.outputPath(),
json_path=json_export_path,
sqconn=self.sqconn)
def stopRecorder(self):
self.recorder.stop()
filename = QtGui.QFileDialog.getSaveFileName(None, 'Save As:', '',
'*.mov ;; *.*')
if filename:
if not str(filename).lower().endswith('.mov'):
filename += '.mov'
self.exporter = Exporter(self.workdir, filename)
self.exporter.start()
print('\nFile saved {f}\n'.format(f=filename))
def process(input_file_name, output_file_name):
exporter = Exporter()
try:
decks = exporter.read_and_convert_decks(input_file_name)
except Exception:
print("Error while reading CSV file:")
raise
try:
exporter.output_decks(decks, output_file_name)
except Exception:
print("Error while writing CSV file:")
raise
return len(decks)
def test_export_orphans(self, mocked_upload, local_db, import_test_data):
with local_db.session() as session:
num = session.query(NPRTable).count()
assert num > 0
exporter = Exporter()
exporter.export_orphans()
filename = "Leeg_NPR_BACKUP.csv"
path = os.path.join(settings.TMP_DATA_DIR, filename)
mocked_upload.assert_called_with(path, filename)
with open(path) as f:
lines = f.readlines()
assert len(lines) == 17
# cleanup
os.remove(path)
def main():
"""
Placeholder
"""
# parse arguments
args = parseArguments()
obj = Exporter()
# for each scene
scenes = getSceneList(args)
for scene in scenes:
# export hdf sub-datasets to geotiff
obj.process(scene)
return
def process_askue():
e = Exporter()
try:
logging.debug('Trying to connect to FTP server...')
with FTP(S.FTP_SERVER, S.FTP_USER, S.FTP_PASSWORD, timeout=5) as fc:
logging.debug('Looking for files in FTP directory')
# Find files and retrieve it
inbox_files = fc.mlsd(S.REMS_PATH)
filenames = [e[0] for e in inbox_files if askue_filename(e[0])]
if not filenames:
logging.info('Inbox directory is empty...')
return
if len(filenames) > 1:
logging.debug(
'More than 1 file were found: {}'.format(
'\n'.join(filenames)))
rfile = max(filenames, key=date_from_filename)
logging.info('Retrieving {}...'.format(rfile))
tf = NamedTemporaryFile()
fc.retrbinary('RETR {}'.format(j(S.REMS_PATH, rfile)), tf.write)
ftp_pos = tf.tell()
try:
if S.APPEND_ON:
lines = (record_to_csv(rec) for rec in
e.get_routes(datetime.now()))
append_lines(tf, lines)
else:
logging.debug(
'Will not append lines (switched off in settings)')
except Exception:
logging.exception(
'Error appending lines to file! Sending as is')
tf.seek(ftp_pos)
tf.truncate()
tf.seek(0)
dest_path = j(S.IOMM_PATH, rfile)
# Send file back to FTP
logging.info('Sending file... {}'.format(dest_path))
fc.storbinary('STOR {}'.format(dest_path), tf)
logging.info('Cleaning up directory...')
for fname in filenames:
filepath = j(S.REMS_PATH, fname)
fc.delete(filepath)
finally:
e.close_connection()
def __init__(self, model, view, defaultPath):
self._view = view
self._model = model
self._path = defaultPath
self._appPath = str(Path(__file__).parent.absolute())
self._exporter = Exporter()
self.appPath = str(Path(__file__).parent.absolute())
self._view.setWindowIcon(QtGui.QIcon(path.join(self._appPath, "NotingLogo.svg")))
# Choose if you want to open or create a new session.
method = self._view.selectSessionDialog()
self._initSession(method)
self._view.listNotes.setModel(self._model)
self._connectSignals()
def main():
"""
Placeholder
"""
# parse arguments
args = parseArguments()
obj = Exporter()
# for each scene
scenes = getSceneList(args)
for scene in scenes:
# generated cloud-masked geotiffs
if os.path.exists(scene):
obj.getImages(scene)
return
def search():
search_term = request.args.get('query','')
if search_term != '':
patient_results = Exporter.search(search_term)
patient_html_data = FhirParser.parse_results_for_html(patient_results)
return render_template('search_results.html', search_term=search_term, results=patient_html_data)
else:
return redirect(url_for('export_page'))
def convert(
self,
input_file,
output_dir=None,
output_format='json',
export_with_errors=False,
errors_file='./files/output/errors.json'
):
# Register custom importers
c_importers = {
'json': json_importer
}
# Register custom exporters
c_exporters = {
'csv': csv_exporter
}
importer = Importer(custom_importers=c_importers)
validator = HotelValidator()
exporter = Exporter(custom_exporters=c_exporters)
try:
data = importer.load(input_file)
data, data_backup = tee(data)
# Validate data
for row in data:
validator.validate(row)
error_count = len(validator.errors)
LOG.warning(f'Validated {validator.count} records. Found {error_count} errors.')
if error_count:
with open(errors_file, 'w') as error_file:
error_file.write(json.dumps(validator.errors))
if export_with_errors:
LOG.debug(f'Export to {output_format} started.')
output = exporter.write(data_backup, output_dir, output_format)
LOG.warning(f'Exported successfully. View file: {os.path.abspath(output)}')
else:
LOG.error(f'Errors identified. Export process stopped. You can view the error logs at {os.path.abspath(errors_file)}')
except Exception as e:
LOG.error(str(e))
def __init__(self,checkRuns={},parent=None):
QObject.__init__(self, parent)
if(not checkRuns):
self.runs = {}
self.regexes = GameDBRegex()
self.dats = []
self.scrapers = []
self.exporter = Exporter()
self.matcher = Matcher()
self.patcher = Patcher('patches.xlsx')
self.database = Database()
def test_log_error(self):
"""don't show a HTTPServer error message"""
e = Exporter
r = MockRequest("")
with Exporter(r, ("", 65536), e) as handler:
with captured_stdout() as stdout, captured_stderr() as stderr:
e.log_error(handler, "test")
# capture
#
self.assertEqual(stdout.getvalue(), "")
self.assertEqual(stderr.getvalue(), "")
def stopRecorder(self):
self.recorder.stop()
filename = QtGui.QFileDialog.getSaveFileName(
None, 'Save As:', '', '*.mov ;; *.*')
if filename:
if not str(filename).lower().endswith('.mov'):
filename += '.mov'
self.exporter = Exporter(self.workdir, filename)
self.exporter.start()
print('\nFile saved {f}\n'.format(f=filename))
def do_conversion(self):
exporter = Exporter()
decks = None
try:
exporter.verify_input_file(self.input_file_name.get())
except Exception as ex:
messagebox.showwarning("Invalid File", ex)
raise
try:
decks = exporter.read_and_convert_decks(self.input_file_name.get())
except Exception as ex:
messagebox.showerror("Could not convert",
"Error while reading CSV file.")
raise
try:
exporter.output_decks(decks, self.output_file_name.get())
except Exception as ex:
messagebox.showerror("Could not convert",
"Error while writing CSV file.")
raise
messagebox.showinfo("Success", f'{len(decks)} decks converted.')
# filter_row/ filter_row_segment is used to filter the data based on the ETL start_date and end_date
if param.reset_time == param.reset_value:
filter_row = " where updated_at >='" + str(
param.reset_start_date) + "' and updated_at<'" + str(
param.reset_end_date) + "'"
else:
filter_row = " where updated_at >='" + str(
param.start_date) + "' and updated_at<'" + str(param.end_date) + "'"
#filter_row = " "
filter_row_segment = " where updated_at::date >= current_date::date -1 and updated_at::date < current_date::date "
if (host == "nwsl"):
param.counter = 2
for i in param.tbl_nwsl:
print('extraction of ' + i + ' started')
runner = Exporter("select * from " + i + filter_row, i + "_nwsl")
runner.start()
runner_employee = Exporter("select * from newsletter_customers",
'newsletter_customers')
runner_employee.start()
elif (host == "core"):
param.counter = 26
for i in param.tbl_core:
print('extraction of ' + i + ' started')
runner = Exporter("select * from " + i + filter_row, i)
runner.start()
runner_employee = Exporter("select * from employees", 'employees')
runner_employee.start()
for thread_id in threads.keys():
cons = get_inters_and_cons(threads[thread_id], edges)
# Update cons
for uid in cons:
if uid in radius:
radius[uid] = radius[uid] + 1
else:
radius[uid] = 1
interactions[group_id] = edges
contributions[group_id] = radius
# OVERALL
all_id = 0
all_name = 'All'
all_edges = Edge.combine(interactions.values())
all_conts = User.combine(contributions.values())
interactions[all_id] = all_edges
contributions[all_id] = all_conts
group_hash[all_id] = all_name
outs = Exporter.export(contributions, user_hash, interactions)
#print outs
success = sql_handler.insert_plots(sys.argv[1], outs, group_hash)
if success == False:
print False
exit()
sql_handler.cleanup()
print True
#doc = xmltodict.parse(fd.read())
class GameDB(QObject):
def __init__(self,checkRuns={},parent=None):
QObject.__init__(self, parent)
if(not checkRuns):
self.runs = {}
self.regexes = GameDBRegex()
self.dats = []
self.scrapers = []
self.exporter = Exporter()
self.matcher = Matcher()
self.patcher = Patcher('patches.xlsx')
self.database = Database()
started = pyqtSignal(str)
progress = pyqtSignal(int)
newmax = pyqtSignal(int)
finished = pyqtSignal(str)
def import_dats(self):
self.dats = []
for xmlfile in os.listdir("DAT"):
dat = DAT()
dat.read_dat(os.path.join("DAT",xmlfile))
self.dats.append(dat)
for dat in self.dats:
print "parsing " + dat.filename
self.import_dat(dat)
self.dats = None
self.import_new_ROMS()
# temp fix to load serial
print "Importing PSP serials"
dat = DAT()
dat.read_dat('libretro-database/metadat/no-intro/Sony - PlayStation Portable.dat')
self.import_softwareflags(dat)
def import_dat(self,dat):
datFileId = None
datGameId = None
datRomId = None
self.regexes.init_regexes(dat.releaseGroup)
sysresult = self.regexes.get_regex_result("System",dat.header["name"])
systemId = self.database.getSystem(sysresult.group('Manufacturer'),sysresult.group('Name'))
datType = 'Standard' if sysresult.group('DatType') == None else sysresult.group('DatType')
datVersion = dat.header['version'] if dat.header['version'] is not None else dat.header['date']
#check if version is a date
datDate = self.regexes.get_cleaned_date(datVersion)
if datDate is not None:
datVersion = datDate
datFileId = self.database.getDATFile(systemId,dat.filename,datType,dat.releaseGroup,datVersion)
for gamekey,gamevalue in dat.softwares.iteritems():
datGameId = self.database.getDATGame(datFileId,gamevalue['Name'],gamevalue['CloneOf'],gamevalue['RomOf'])
for rom in gamevalue['Roms']:
datRomId = self.database.getDATROM(datFileId,datGameId,rom['name'],rom['merge'],rom['size'],rom['crc'],rom['md5'],rom['sha1'])
self.database.save()
def import_new_ROMS(self):
systemId = None
releaseGroup = None
softwareId = None
releaseId = None
romId = None
datRoms = self.database.getNewRoms()
for datRom in datRoms:
#get releaseGroup regexes
if releaseGroup != datRom[1]:
releaseGroup = datRom[1]
self.regexes.init_regexes(releaseGroup)
#get system
if systemId != datRom[0]:
systemId = datRom[0]
print "exporting new roms for " + self.database.getSystemName(systemId)
#get software
gameName = datRom[2] if datRom[3] == '' else datRom[3]
softwareId = self.import_software(gameName,systemId)
#get release
releaseName = datRom[2]
releaseId = self.import_release(releaseName,softwareId)
#release flags
self.import_releaseflags(releaseName,releaseId)
romId = self.database.getROM(releaseId,*datRom[5:])
self.database.save()
def import_software(self,gameName,systemId):
softresult = self.regexes.get_regex_result("Software",gameName)
softwarename = softresult.group('Name')
softwaretype = 'BIOS' if softresult.group('BIOS') is not None else softresult.group('Type') if softresult.group('Type') is not None else 'Game'
return self.database.getSoftware(systemId,softwarename,softwaretype)
def import_release(self,releaseName,softwareId):
compresult = self.regexes.get_regex_result("Compilation",releaseName)
devstatusresult = self.regexes.get_regex_result("DevStatus",releaseName)
demoresult = self.regexes.get_regex_result("Demo",releaseName)
licenseresult = self.regexes.get_regex_result("License",releaseName)
if compresult is not None:
releaseType = compresult.group('Compilation')
elif devstatusresult is not None:
releaseType = devstatusresult.group('DevStatus')
elif demoresult is not None:
releaseType = demoresult.group('Demo')
elif licenseresult is not None:
releaseType = licenseresult.group('License')
else:
releaseType = 'Commercial'
return self.database.getRelease(releaseName,releaseType,softwareId)
def import_softwareflags(self,dat):
for gamekey,gamevalue in dat.softwares.iteritems():
sysresult = self.regexes.get_regex_result("System",dat.header["name"])
systemId = self.database.getSystem(sysresult.group('Manufacturer'),sysresult.group('Name'))
softwareId = self.import_software(gamevalue['Name'],systemId)
for rom in gamevalue['Roms']:
releaseId = self.import_release(gamevalue['Name'],softwareId)
if 'Serial' in gamevalue:
self.database.addReleaseFlagValue(releaseId,self.database.getReleaseFlag('ProductID'),gamevalue['Serial'])
self.database.save()
def import_releaseflags(self,releaseName,releaseId):
for regionresult in self.regexes.get_regex_results("Region",releaseName):
self.database.addReleaseFlagValue(releaseId,self.database.getReleaseFlag('Region'),regionresult.group('Region'))
for languageresult in self.regexes.get_regex_results("Language",releaseName):
self.database.addReleaseFlagValue(releaseId,self.database.getReleaseFlag('Language'),languageresult.group('Language'))
versionresult = self.regexes.get_regex_result("Version",releaseName)
if(versionresult):
self.database.addReleaseFlagValue(releaseId,self.database.getReleaseFlag('Version'),versionresult.group('Version'))
revisionresult = self.regexes.get_regex_result("Revision",releaseName)
if(revisionresult):
self.database.addReleaseFlagValue(releaseId,self.database.getReleaseFlag('Revision'),revisionresult.group('Revision'))
baddumpresult = self.regexes.get_regex_result("DumpStatus",releaseName)
if(baddumpresult):
self.database.addReleaseFlagValue(releaseId,self.database.getReleaseFlag('BadDump'),baddumpresult.group('BadDump'))
def import_scrapers(self):
self.scrapers = []
scrapersfile = io.open('Scrapers/scrapers.csv','r',encoding='utf-8')
for scraperline in scrapersfile:
scraperCols = scraperline.split(';')
scraperId = self.database.getScraper(*scraperCols)
scraper = Scraper(*scraperCols)
for scraperSystemKey,scraperSystem in scraper.systems.items():
print "exporting game data for " + scraper.name + " - " + scraperSystemKey
scraperSystemId = self.database.getScraperSystem(scraperId,scraperSystem['systemName'],scraperSystem['systemAcronym'],scraperSystem['systemURL'])
for game in scraperSystem['systemGames'].itervalues():
scraperGameId = self.database.getScraperGame(scraperSystemId,game['gameName'],game['gameUrl'])
if game['gameParsed']=='Yes':
for flag in game['softwareFlags']:
self.database.addScraperGameFlagValue(scraperGameId,flag['name'],flag['value'])
for release in game['releases']:
scraperReleaseId = self.database.getScraperRelease(scraperGameId,release['name'],release['region'],release['type'])
for flag in release['releaseFlags']:
self.database.addScraperReleaseFlagValue(scraperReleaseId,flag['name'],flag['value'])
for image in release['releaseImages']:
scraperReleaseImageId = self.database.getScraperReleaseImage(scraperReleaseId,image['name'],image['type'])
self.database.save()
def match_systems(self):
for synonym in self.matcher.synonyms:
self.database.addSynonym(synonym['key'],synonym['value'],synonym['type'])
self.database.matchSystemScraperSystem()
self.database.save()
def match_softwares(self):
systems = self.database.getMappedSystems(1) ## scraperId 1 - GameFaqs
for system in systems:
print "Matching Softwares for System : " + system[1]
releasegamelist = self.database.getScraperRelease2GameList(system[2])
releaseDic = {r[0]:r[1] for r in releasegamelist}
gameDic = {r[0]:r[2] for r in releasegamelist}
softwares = self.database.getSoftwareList(system[0])
for software in softwares:
scraperReleaseId = self.matcher.match_fuzzy(releaseDic,software[1],"Full",80)
if scraperReleaseId == None:
scraperReleaseId = self.matcher.match_fuzzy(releaseDic,software[1],"Partial",86)
if scraperReleaseId != None:
self.database.addSoftwareMatch(software[0],gameDic[scraperReleaseId])
self.database.save()
def match_releases(self):
systems = self.database.getMappedSystems(1) ## scraperId 1 - GameFaqs
for system in systems:
print "Matching Releases for System : " + system[1]
releaserows = self.database.getScraperReleaseList(system[0])
for releaserow in releaserows:
matches = self.database.getScraperGame2ReleaseList(releaserow[2],releaserow[3])
if len(matches) == 1:
self.database.addReleaseMatch(releaserow[0],matches[0][0])
elif len(matches) > 1:
releaseDic = {m[0]:m[1] for m in matches}
## clean dat release Name to match it to scraper releaseName
releaseName = self.regexes.get_regex_result('Software',releaserow[1]).group("Name")
scraperReleaseId = self.matcher.match_fuzzy(releaseDic,releaseName,"Full",80)
self.database.addReleaseMatch(releaserow[0],scraperReleaseId)
self.database.save()
def match_software_flags(self):
print "Importing software flags"
softwareflags = self.database.getSoftwareFlagList()
for flagrow in softwareflags:
flagid = self.database.getSoftwareFlag(flagrow[1])
flagvalue = ""
if flagrow[1]=="Developer":
flagvalue = self.database.getSynonym(flagrow[2],'Developer')
flagvalue = self.regexes.get_cleaned_developer(flagvalue)
elif flagrow[1]=="Genre":
flagvalue = self.database.getSynonym(flagrow[2],'Genre')
elif flagrow[1]=="Franchise":
flagvalue = self.database.getSynonym(flagrow[2],'Franchise')
if flagvalue != "" and flagvalue is not None:
self.database.addSoftwareFlagValue(flagrow[0],flagid,flagvalue)
self.database.save()
def match_release_flags(self):
print "Importing release flags"
releaseflags = self.database.getReleaseFlagList()
for flagrow in releaseflags:
flagid = self.database.getReleaseFlag(flagrow[1])
flagvalue = ""
if flagrow[1]=="Publisher":
flagvalue = self.database.getSynonym(flagrow[2],'Developer')
flagvalue = self.regexes.get_cleaned_developer(flagvalue)
elif flagrow[1]=="ReleaseDate":
flagvalue = self.regexes.get_cleaned_date(flagrow[2])
elif flagrow[1]=="ProductID":
flagvalue = flagrow[2]
elif flagrow[1]=="BarCode":
flagvalue = flagrow[2]
if flagvalue != "" and flagvalue is not None:
self.database.addReleaseFlagValue(flagrow[0],flagid,flagvalue)
self.database.save()
def export_gamedbflags(self):
lstFlags = [('Developer','developer','software'), \
('Franchise','franchise','software'), \
('Genre','genre','software'), \
('Publisher','publisher','release'), \
('ProductID','serial','release'), \
('ReleaseDate','releasemonth','release'), \
('ReleaseDate','releaseyear','release')]
for flagtuple in lstFlags:
flag = {}
flag['srcName'] = flagtuple[0]
flag['destName'] = flagtuple[1]
flag['systems'] = []
systemrows = self.database.getSystemDic()
for systemId, systemName in systemrows.iteritems():
print "Exporting {0} flag {1} for system {2}".format(flagtuple[2],flagtuple[1],systemName)
system = {}
system['name'] = systemName
system['roms'] = []
rows = self.database.getSystemFlagValues(systemId,flag['srcName'])
if systemName in ['Sony - PlayStation Portable','Sony - PlayStation']:
rows = [r for r in rows if r[1] is not None]
for row in rows:
rom = {}
rom['name'] = row[2]
if systemName in ['Sony - PlayStation Portable','Sony - PlayStation']:
rom['key'] = 'serial'
rom['keyvalue'] = '"' + row[1] + '"'
else:
rom['key'] = 'crc'
rom['keyvalue'] = row[0]
if flag['destName']=="releasemonth":
rom['flagvalue'] = str(datetime.strptime(row[3],'%Y-%m-%d %H:%M:%S').month)
elif flag['destName']=="releaseyear":
rom['flagvalue'] = str(datetime.strptime(row[3],'%Y-%m-%d %H:%M:%S').year)
else:
rom['flagvalue'] = row[3]
system['roms'].append(rom)
if len(system['roms']) > 0:
system['roms'].sort(key=lambda x: x["name"])
flag['systems'].append(system)
self.exporter.export_rdb_dat(flag)
def export_rdbs(self):
systemrows = self.database.getSystemDic()
for systemId,systemname in systemrows.iteritems():
print "Exporting rdb for " + systemname
if systemname in ['Sony - PlayStation Portable','Sony - PlayStation']:
key = 'serial'
else:
key = 'rom.crc'
self.exporter.create_rdb(systemname,key)
def apply_patches(self,stage):
patchname = "patch_" + stage + ".sql"
self.patcher.GenerateScript(patchname,stage)
self.database.run_script(patchname)
print "Patch " + stage + " applied"
class NotingCtrl:
def __init__(self, model, view, defaultPath):
self._view = view
self._model = model
self._path = defaultPath
self._appPath = str(Path(__file__).parent.absolute())
self._exporter = Exporter()
self.appPath = str(Path(__file__).parent.absolute())
self._view.setWindowIcon(QtGui.QIcon(path.join(self._appPath, "NotingLogo.svg")))
# Choose if you want to open or create a new session.
method = self._view.selectSessionDialog()
self._initSession(method)
self._view.listNotes.setModel(self._model)
self._connectSignals()
def _initSession(self, method):
"""Function to initialize the session opening or creating a new one."""
if method == "open":
self._openSession()
elif method == "new":
self._newSession()
self._model.setSessionPath(self.sessionPath)
self._model.initSession()
#Disables the textEdit and move buttons until a note is created or opened.
self._view.textEdit.setDisabled(True)
self._view.textEdit.clear()
self._view.upButton.setDisabled(True)
self._view.downButton.setDisabled(True)
self._view.infoText.setText('Session name: {}\nNo openned note'.format(
self._model.sessionInfo['name']))
def _openSession(self):
# Return the session path from a file selector.
self.sessionPath = self._view.openSessionDialog()
def _newSession(self):
# Ask for the name of the session.
sessionName = self._view.nameNewSessionForm()
if not path.splitext(sessionName)[-1] == '.json':
sessionName = sessionName + '.json'
self.sessionPath = path.join(self._path, sessionName)
def _connectSignals(self):
"""Manage the signal connection with controller methods."""
# Data list
self._view.listNotes.doubleClicked.connect(self.openNote)
self._view.returnKeyPressed.connect(self.openNote)
# Up/down buttons
self._view.upButton.clicked.connect(lambda: self.moveNote('up'))
self._view.downButton.clicked.connect(lambda: self.moveNote('down'))
# Create notes shortcuts
self._view.createNoteShortcut.activated.connect(lambda: self.createNote('note'))
self._view.createQuestionShortcut.activated.connect(lambda: self.createNote('question'))
self._view.createTaskShortcut.activated.connect(lambda: self.createNote('task'))
# Menu Bar connections
self._view.newSession.triggered.connect(lambda: self._initSession("new"))
self._view.openSession.triggered.connect(lambda: self._initSession("open"))
self._view.saveSession.triggered.connect(self.saveSession)
self._view.saveSession.setShortcut(self._view.saveSessionShortcut)
self._view.exportPDF.triggered.connect(self.exportPDF)
# Management
self._view.saveNoteShortcut.activated.connect(self.saveNoteText)
self._view.focusListShortcut.activated.connect(self.focusList)
def openNote(self, idx):
"""Open a note when it's clicked or the return key is used."""
note = self._model.openNote(idx)
self._updateNoteInfo(note)
self._view.textEdit.setText(note.text)
self._view.textEdit.setEnabled(True)
self._view.upButton.setEnabled(True)
self._view.downButton.setEnabled(True)
self._view.textEdit.setFocus()
def createNote(self, noteType):
"""Create a new note TODO: AND OPENS IT."""
noteName = self._view.newNoteForm(noteType)
if noteName == None:
return
self._view.textEdit.setText('')
newNote = Note(noteName, noteType, '')
self._model.addNote(newNote)
self._updateNoteInfo(newNote)
self._view.listNotes.setCurrentIndex(self._model.currentIndex)
self._view.textEdit.setEnabled(True)
self._view.upButton.setEnabled(True)
self._view.downButton.setEnabled(True)
self._view.textEdit.setFocus()
def moveNote(self, direction):
# TODO: Change to drag and drop at some point.
self._model.swapNote(direction)
self._view.listNotes.setCurrentIndex(self._model.currentIndex)
def saveNoteText(self):
"""Save note text. TODO: Make available to rename note."""
noteText = self._view.textEdit.toPlainText()
self._model.saveNoteText(noteText)
def saveSession(self):
"""Parses the sesion to the JSON file."""
self._model.saveSession()
def _updateNoteInfo(self, note):
"""Update the note info in the infoText. TODO: Separate session and note info."""
self._view.infoText.setText('Session name: {}\nName: {}\nType: {}\n'.format(
self._model.sessionInfo['name'], note.name, note.noteType))
def focusList(self):
"""Focus list when called."""
self._view.listNotes.setFocus()
def exportPDF(self):
"""Function to export the session in PDF format."""
ordered = self._view.selectExportOrderForm()
path = self._view.exportSessionDialog()
self._exporter.export_pdf(self._appPath, path, self._model.sessionInfo, self._model.getNotes(), ordered)
parser.add_argument(
'-e',
'--enddate',
type=int,
default=time.strftime('%Y%m%d', time.localtime(time.time())),
help='end date (YYYYMMDD), if not assigned, set as today')
parser.add_argument(
'-t',
'--threads',
type=int,
default=multiprocessing.cpu_count(),
help=
'how many threads to use, if not assigned thread will be the same as cpu cores'
)
parser.add_argument('-p',
'--province',
help='the province to get, if not assigned crawl all')
args = parser.parse_args()
logger = init_logger()
db = DBIO(args.database)
ex = Exporter(db, args.threads, args.startdate, args.enddate)
if args.province:
ex.get_province(args.province)
else:
for prov in provinces:
ex.get_province(prov)
db.close()
e_time = time.time()
logger.info('Used %s seconds' % (e_time - s_time))
import sys
import os
# argument passed to runner program is stored in host variable
host = sys.argv[1]
#host = "cs"
param.dbconn(host)
if not os.path.exists(param.newpath):
os.makedirs(param.newpath)
if sys.argv[1] in ('cs', 'abasence.io'):
print("Extracting Json for " + str(param.start_date) + " - " +
str(param.end_date))
else:
print("No other ETL defiend yet")
if (host == "cs"):
param.counter = 2
for i in param.tbl_cs:
print('Extracting Json for ' + i + ' started')
runner = Exporter(i)
runner.start()
else:
print("no other mongo source is defined yet")
# run the ETL process until all the mentioned collections in the param file are exported.
while param.counter != 0:
importer.import_data()
class Gui(QtGui.QWidget):
# =========================================================================
def __init__(self, parent=None):
super(Gui, self).__init__(parent)
self.videoComboBox = QtGui.QComboBox(self)
self.videoComboBox.currentIndexChanged.connect(self.videoIndexChanged)
self.audioComboBox = QtGui.QComboBox(self)
self.audioComboBox.currentIndexChanged.connect(self.audioIndexChanged)
self.recordPushButton = QtGui.QPushButton(self)
self.recordPushButton.setCheckable(True)
self.recordPushButton.clicked.connect(self.recordClicked)
separator = QtGui.QFrame(self)
separator.setFrameStyle(QtGui.QFrame.HLine | QtGui.QFrame.Sunken)
layout = QtGui.QVBoxLayout(self)
layout.addWidget(self.videoComboBox)
layout.addWidget(self.audioComboBox)
layout.addWidget(separator)
layout.addWidget(self.recordPushButton)
self.setLayout(layout)
self.resetToDefault()
# =========================================================================
def resetToDefault(self):
self.video = Video()
self.audio = Audio()
self.populateVideoComboBox()
self.populateAudioComboBox()
self.recordPushButton.setIcon(
QtGui.QIcon(os.path.join(scriptDir, 'rec_btn_off.svg')))
self.recordPushButton.setChecked(False)
self.recordPushButton.setEnabled(False)
# =========================================================================
def populateVideoComboBox(self):
self.videoComboBox.clear()
self.videoComboBox.addItem('Select video screen...')
for screen in self.video.screens:
comboBoxItem = 'Screen {i}, size {s}'.format(
i=self.videoComboBox.count(), s=screen.size)
self.videoComboBox.addItem(comboBoxItem)
# =========================================================================
def populateAudioComboBox(self):
self.audioComboBox.clear()
self.audioComboBox.addItem('Select audio device...')
for device in self.audio.devices:
comboBoxItem = '{d}'.format(d=device.description)
self.audioComboBox.addItem(comboBoxItem)
# =========================================================================
def videoIndexChanged(self, index):
if index:
self.video.screenSet(index - 1)
self.checkEnableRecord()
# =========================================================================
def audioIndexChanged(self, index):
if index:
self.audio.deviceSet(index - 1)
self.checkEnableRecord()
# =========================================================================
def recordClicked(self):
if self.recordPushButton.isChecked():
self.videoComboBox.setEnabled(False)
self.audioComboBox.setEnabled(False)
self.recordPushButton.setIcon(
QtGui.QIcon(os.path.join(scriptDir, 'rec_btn_on.svg')))
self.startRecorder()
else:
self.videoComboBox.setEnabled(True)
self.audioComboBox.setEnabled(True)
self.recordPushButton.setIcon(
QtGui.QIcon(os.path.join(scriptDir, 'rec_btn_off.svg')))
self.stopRecorder()
# =========================================================================
def checkEnableRecord(self):
if self.videoComboBox.currentIndex() and \
self.audioComboBox.currentIndex():
self.recordPushButton.setEnabled(True)
else:
self.recordPushButton.setEnabled(False)
# =========================================================================
def startRecorder(self):
strftime = time.strftime('%Y%m%d%H%M%S')
self.workdir = '/tmp/xcorder/{s}/'.format(s=strftime)
if not os.path.isdir(self.workdir):
os.makedirs(self.workdir)
self.recorder = Recorder(self.video.screen, self.audio.device,
self.workdir)
self.recorder.start()
# =========================================================================
def stopRecorder(self):
self.recorder.stop()
filename = QtGui.QFileDialog.getSaveFileName(None, 'Save As:', '',
'*.mov ;; *.*')
if filename:
if not str(filename).lower().endswith('.mov'):
filename += '.mov'
self.exporter = Exporter(self.workdir, filename)
self.exporter.start()
print('\nFile saved {f}\n'.format(f=filename))
# =========================================================================
def infoDialog(self, message):
messageBox = QtGui.QMessageBox(QtGui.QMessageBox.Information,
'Message',
message,
QtGui.QMessageBox.Ok,
parent=self)
messageBox.show()
from project_creator import ProjectCreator
from exporter import Exporter
if __name__ == '__main__':
default_project_dir = os.getcwd()
parser = argparse.ArgumentParser()
parser.add_argument('-export',
help='Export definitions into common header file',
action='store_true')
parser.add_argument('-create',
help='Create a project file',
action='store_true')
parser.add_argument('--build-system',
help='Build system that is used for project',
dest='build_system',
default=None)
parser.add_argument('--project-dir',
help='A project directory for analysis',
dest='project_dir',
default=default_project_dir)
args = parser.parse_args()
if args.create:
pc = ProjectCreator(args.project_dir, args.build_system)
pc.create_project_file()
elif args.export:
exporter = Exporter(args.project_dir)
exporter.export()
#--------------------------------------------------
# data extraction
parser = argparse.ArgumentParser()
parser.add_argument('input_file')
parser.add_argument('output_file')
parser.add_argument('--fine', dest='mode', action='store_const', const=Exporter.finemode, default=Exporter.rawmode,
help = 'sends output to output.html for better readability' )
arg = parser.parse_args()
p = TranscriptGen(arg.input_file)
transcripts = p.extract()
# default output is rawmode
exp = Exporter(arg.mode, arg.output_file)
i = 0
bpcount = 0
gecount = 0
agcount = 0
tpcount = 0
pucount = 0
brcount = 0
wecount = 0
hecount = 0
bmcount = 0
o2count = 0
for transcript in transcripts:
bp = str(getBloodP(transcript))
ge = str(getGender(transcript))
ag = str(getAge(transcript))
for thread_id in threads.keys():
cons = get_inters_and_cons(threads[thread_id], edges)
# Update cons
for uid in cons:
if uid in radius:
radius[uid] = radius[uid] + 1
else:
radius[uid] = 1
interactions[group_id] = edges
contributions[group_id] = radius
# OVERALL
all_id = 0
all_name = 'All'
all_edges = Edge.combine(interactions.values())
all_conts = User.combine(contributions.values())
interactions[all_id] = all_edges
contributions[all_id] = all_conts
group_hash[all_id] = all_name
outs = Exporter.export(contributions, user_hash, interactions)
#print outs
success = sql_handler.insert_plots(sys.argv[1], outs, group_hash)
if success == False:
print False
exit()
sql_handler.cleanup()
print True
#doc = xmltodict.parse(fd.read())
def export_patient():
patient_id = request.args.get('pt_id','')
exporter = Exporter(patient_id)
page_resource = exporter.export_page_resource()
return render_template('export_page.html', patient=page_resource, id=patient_id)
import sys
from api import Api
from exporter import Exporter
from datetime import datetime, timedelta
if __name__ == '__main__':
api = Api()
exporter = Exporter()
producttypes = api.getProductTypes()
products = api.getProducts()
exporter.productExport(products, producttypes)
daysToSubtract = 0
if len(sys.argv) > 1:
daysToSubtract = sys.argv[1]
dateStr = (datetime.today() - timedelta(days=int(daysToSubtract))).strftime('%Y-%m-%d')
prices = api.getPricesFromDate(dateStr)
exporter.exportPrices(prices, dateStr)
async def home(user: str):
exporter = Exporter(user)
if not await exporter.valid_user():
return {"error": "invalid user"}
return {"hello": user}
def __init__(self, args,
file_class=File,
chunk_class=Chunk,
segment_class=Segment,
peer_class=Peer):
if hasattr(self, "_initialized") and self._initialized:
return
self.file_class = file_class
self.chunk_class = chunk_class
self.segment_class = segment_class
self.peer_class = peer_class
self.args = args
self.sync = args.sync
self.width = args.width
self.height = args.height
self.margin = args.margin
self.show_fps = args.show_fps
self.export = args.export
self.capture_message_log = args.capture_message_log
self.play_message_log = args.play_message_log
self.waveform_gain = args.waveform_gain
self._standalone = args.standalone
self._target_aspect_ratio = self._get_aspect_ratio_from_args()
self.logger = logging.getLogger("visualizer")
self.reset()
self._frame_count = 0
self.exiting = False
self.time_increment = 0
self.stopwatch = Stopwatch()
self._synth_instance = None
self._synth_port = None
self._synced = False
self._layers = []
self._warned_about_missing_pan_segment = False
self.gl_display_mode = GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH
self._accum_enabled = False
self._3d_enabled = False
self.fovy = 45
self.near = 0.1
self.far = 100.0
self._fullscreen = False
self._text_renderer_class = getattr(text_renderer_module, TEXT_RENDERERS[args.text_renderer])
if args.camera_script:
self._camera_script = CameraScriptInterpreter(args.camera_script)
else:
self._camera_script = None
if self.show_fps:
self.fps_history = collections.deque(maxlen=10)
self.previous_shown_fps_time = None
if not args.standalone:
if args.port:
port = args.port
else:
port = self._get_orchestra_port()
self.orchestra_host = args.host
self.orchestra_port = port
self.setup_osc()
self.orchestra.register(self.server.port)
self._screen_dumper = Exporter(".", self.margin, self.margin, self.width, self.height)
if self.export:
self.export_fps = args.export_fps
import shutil
if args.export_dir:
export_dir = args.export_dir
elif hasattr(args, "sessiondir"):
export_dir = "%s/rendered_%s" % (args.sessiondir, self.__class__.__name__)
else:
export_dir = "export"
if os.path.exists(export_dir):
shutil.rmtree(export_dir)
os.mkdir(export_dir)
self.exporter = Exporter(export_dir, self.margin, self.margin, self.width, self.height)
if self.play_message_log:
self._message_log_reader = MessageLogReader(self.play_message_log)
if self.capture_message_log:
self._message_log_writer = MessageLogWriter(self.capture_message_log)
self._audio_capture_start_time = None
self._initialized = True
class Gui(QtGui.QWidget):
# =========================================================================
def __init__(self, parent=None):
super(Gui, self).__init__(parent)
self.videoComboBox = QtGui.QComboBox(self)
self.videoComboBox.currentIndexChanged.connect(self.videoIndexChanged)
self.audioComboBox = QtGui.QComboBox(self)
self.audioComboBox.currentIndexChanged.connect(self.audioIndexChanged)
self.recordPushButton = QtGui.QPushButton(self)
self.recordPushButton.setCheckable(True)
self.recordPushButton.clicked.connect(self.recordClicked)
separator = QtGui.QFrame(self)
separator.setFrameStyle(QtGui.QFrame.HLine|QtGui.QFrame.Sunken)
layout = QtGui.QVBoxLayout(self)
layout.addWidget(self.videoComboBox)
layout.addWidget(self.audioComboBox)
layout.addWidget(separator)
layout.addWidget(self.recordPushButton)
self.setLayout(layout)
self.resetToDefault()
# =========================================================================
def resetToDefault(self):
self.video = Video()
self.audio = Audio()
self.populateVideoComboBox()
self.populateAudioComboBox()
self.recordPushButton.setIcon(QtGui.QIcon(os.path.join(scriptDir, 'rec_btn_off.svg')))
self.recordPushButton.setChecked(False)
self.recordPushButton.setEnabled(False)
# =========================================================================
def populateVideoComboBox(self):
self.videoComboBox.clear()
self.videoComboBox.addItem('Select video screen...')
for screen in self.video.screens:
comboBoxItem = 'Screen {i}, size {s}'.format(
i=self.videoComboBox.count(), s=screen.size)
self.videoComboBox.addItem(comboBoxItem)
# =========================================================================
def populateAudioComboBox(self):
self.audioComboBox.clear()
self.audioComboBox.addItem('Select audio device...')
for device in self.audio.devices:
comboBoxItem = '{d}'.format(d=device.description)
self.audioComboBox.addItem(comboBoxItem)
# =========================================================================
def videoIndexChanged(self, index):
if index:
self.video.screenSet(index - 1)
self.checkEnableRecord()
# =========================================================================
def audioIndexChanged(self, index):
if index:
self.audio.deviceSet(index - 1)
self.checkEnableRecord()
# =========================================================================
def recordClicked(self):
if self.recordPushButton.isChecked():
self.videoComboBox.setEnabled(False)
self.audioComboBox.setEnabled(False)
self.recordPushButton.setIcon(QtGui.QIcon(os.path.join(scriptDir, 'rec_btn_on.svg')))
self.startRecorder()
else:
self.videoComboBox.setEnabled(True)
self.audioComboBox.setEnabled(True)
self.recordPushButton.setIcon(QtGui.QIcon(os.path.join(scriptDir, 'rec_btn_off.svg')))
self.stopRecorder()
# =========================================================================
def checkEnableRecord(self):
if self.videoComboBox.currentIndex() and \
self.audioComboBox.currentIndex():
self.recordPushButton.setEnabled(True)
else:
self.recordPushButton.setEnabled(False)
# =========================================================================
def startRecorder(self):
strftime = time.strftime('%Y%m%d%H%M%S')
self.workdir = '/tmp/xcorder/{s}/'.format(s=strftime)
if not os.path.isdir(self.workdir):
os.makedirs(self.workdir)
self.recorder = Recorder(
self.video.screen, self.audio.device, self.workdir)
self.recorder.start()
# =========================================================================
def stopRecorder(self):
self.recorder.stop()
filename = QtGui.QFileDialog.getSaveFileName(
None, 'Save As:', '', '*.mov ;; *.*')
if filename:
if not str(filename).lower().endswith('.mov'):
filename += '.mov'
self.exporter = Exporter(self.workdir, filename)
self.exporter.start()
print('\nFile saved {f}\n'.format(f=filename))
# =========================================================================
def infoDialog(self, message):
messageBox = QtGui.QMessageBox( QtGui.QMessageBox.Information,
'Message', message, QtGui.QMessageBox.Ok, parent=self)
messageBox.show()
for j,o in enumerate(candidate): if o.jo in [r.jo for r in bestsubperm]: candidate[j] = bestsubperm[i] i+=1 candidates.append(candidate) return candidates def neighbours_idle(self, size): candidates = [] fororders = ForSchedule() self.UpdateTHpanel() if not self.SignalCheck(): return [] return [self.orders] if __name__ == '__main__': from kivymd.button import * from kivymd.snackbar import Snackbar from kivymd.theming import ThemeManager from kivymd.progressbar import MDProgressBar from kivymd.selectioncontrols import MDCheckbox from kivymd.navigationdrawer import NavigationDrawerIconButton from kivymd.date_picker import MDDatePicker from uix.jscolor import * from uix.jslabel import ContentLabel, TitleLabel from allkv import * multiprocessing.freeze_support() ex = Exporter() app = MDScheduler() app.run()
param.exported_file = dict((el, 0) for el in tbl_all)
param.truncate_tbl = copy.copy(truncate_table)
if (host in param.sources):
fetch_table(host)
param.counter = len(param.tbl_source) + len(param.tbl_source_truncate)
param.redshift_counter = len(param.tbl_source) + len(
param.tbl_source_truncate)
for i in param.tbl_source:
if i in param.truncate_tbl:
runner = Exporter(
"select * from {0}.".format(param.schema) + i + filter_refresh,
param.tbl_source_rename[i])
runner.start()
if i in param.tbl_source_rename:
runner = Exporter(
"select * from {0}.".format(param.schema) + i + filter_row,
param.tbl_source_rename[i])
runner.start()
else:
runner = Exporter(
"select * from {0}.".format(param.schema) + i + filter_row, i)
runner.start()
for j in param.tbl_source_truncate:
def update(c):
asyncio.run(Exporter().generate_achievement_export())
def main():
init_logging()
init_filesystem()
arguments = docopt(__doc__, version='gtfs-exporter %s' % version)
provider_type = arguments['--provider']
provider = DataProvider()
if provider_type == "file":
provider = FileDataProvider(
arguments['--file'],
feed_id=arguments['--id'],
lenient=arguments['--lenient'],
disable_normalization=arguments['--disablenormalize'])
elif provider_type == "url":
provider = HttpDataProvider(
arguments['--url'],
feed_id=arguments['--id'],
lenient=arguments['--lenient'],
disable_normalization=arguments['--disablenormalize'])
elif provider_type == "api":
builder = ApiBuilder(
arguments['--url'],
feed_id=arguments['--id'],
lenient=arguments['--lenient'],
disable_normalization=arguments['--disablenormalize'])
provider = builder.build()
exporter = Exporter(arguments)
sg = ShapeGenerator(
"https://download.geofabrik.de/europe/romania-latest.osm.bz2",
out_path)
# flow needs to be different when receiving data from api
# - load
# - process
# - generate initial gtfs files
# - generate shapes for gtfs
# - generate bundle
# for zip, url
# - generation of shapes
# - load all the feed to process & interpolate
# - generate feed (bundle)
if provider.is_from_api():
exporter.load(provider)
exporter.process()
exporter.export(bundle=False)
sg.generate()
from exporter.util.storage import generate_gtfs_bundle
generate_gtfs_bundle(out_path, bundle=f"gtfs-{arguments['--id']}.zip")
else:
sg.generate()
exporter.load(provider)
exporter.process()
exporter.export(bundle=True)
rg = ReleaseGenerator(GH_REPO, GH_TOKEN)
rg.generate([
os.path.join(out_path, f"gtfs-{arguments['--id']}.zip"),
] + glob.glob(os.path.join(out_path, "*.json")))
class Visualizer:
def __init__(self, args,
file_class=File,
chunk_class=Chunk,
segment_class=Segment,
peer_class=Peer):
if hasattr(self, "_initialized") and self._initialized:
return
self.file_class = file_class
self.chunk_class = chunk_class
self.segment_class = segment_class
self.peer_class = peer_class
self.args = args
self.sync = args.sync
self.width = args.width
self.height = args.height
self.margin = args.margin
self.show_fps = args.show_fps
self.export = args.export
self.capture_message_log = args.capture_message_log
self.play_message_log = args.play_message_log
self.waveform_gain = args.waveform_gain
self._standalone = args.standalone
self._target_aspect_ratio = self._get_aspect_ratio_from_args()
self.logger = logging.getLogger("visualizer")
self.reset()
self._frame_count = 0
self.exiting = False
self.time_increment = 0
self.stopwatch = Stopwatch()
self._synth_instance = None
self._synth_port = None
self._synced = False
self._layers = []
self._warned_about_missing_pan_segment = False
self.gl_display_mode = GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH
self._accum_enabled = False
self._3d_enabled = False
self.fovy = 45
self.near = 0.1
self.far = 100.0
self._fullscreen = False
self._text_renderer_class = getattr(text_renderer_module, TEXT_RENDERERS[args.text_renderer])
if args.camera_script:
self._camera_script = CameraScriptInterpreter(args.camera_script)
else:
self._camera_script = None
if self.show_fps:
self.fps_history = collections.deque(maxlen=10)
self.previous_shown_fps_time = None
if not args.standalone:
if args.port:
port = args.port
else:
port = self._get_orchestra_port()
self.orchestra_host = args.host
self.orchestra_port = port
self.setup_osc()
self.orchestra.register(self.server.port)
self._screen_dumper = Exporter(".", self.margin, self.margin, self.width, self.height)
if self.export:
self.export_fps = args.export_fps
import shutil
if args.export_dir:
export_dir = args.export_dir
elif hasattr(args, "sessiondir"):
export_dir = "%s/rendered_%s" % (args.sessiondir, self.__class__.__name__)
else:
export_dir = "export"
if os.path.exists(export_dir):
shutil.rmtree(export_dir)
os.mkdir(export_dir)
self.exporter = Exporter(export_dir, self.margin, self.margin, self.width, self.height)
if self.play_message_log:
self._message_log_reader = MessageLogReader(self.play_message_log)
if self.capture_message_log:
self._message_log_writer = MessageLogWriter(self.capture_message_log)
self._audio_capture_start_time = None
self._initialized = True
def _get_aspect_ratio_from_args(self):
w, h = map(float, self.args.aspect.split(":"))
return w / h
def _get_orchestra_port(self):
if self.args.host == "localhost":
return self._read_port_from_disk()
else:
return self._read_port_from_network_share()
def _read_port_from_disk(self):
self._read_port_from_file("server_port.txt")
def _read_port_from_file(self, filename):
f = open(filename, "r")
line = f.read()
port = int(line)
f.close()
return port
def _read_port_from_network_share(self):
if platform.system() == "Linux":
return self._read_port_with_unix_smbclient()
elif platform.system() == "Windows":
return self._read_port_via_windows_samba_access()
else:
raise Exception("don't know how to handle your OS (%s)" % platform.system())
def _read_port_with_unix_smbclient(self):
subprocess.call(
'smbclient -N \\\\\\\\%s\\\\TorrentialForms -c "get server_port.txt server_remote_port.txt"' % self.args.host,
shell=True)
return self._read_port_from_file("server_remote_port.txt")
def _read_port_via_windows_samba_access(self):
return self._read_port_from_file(
'\\\\%s\\TorrentialForms\\server_port.txt' % self.args.host)
def reset(self):
self.files = {}
self.peers = {}
self.peers_by_addr = {}
self._segments_by_id = {}
self.torrent_length = 0
self.torrent_title = ""
self.torrent_download_completion_time = None
self.num_segments = 0
self.num_received_segments = 0
self._notified_finished = False
def enable_3d(self):
self._3d_enabled = True
def run(self):
self.window_width = self.width + self.margin*2
self.window_height = self.height + self.margin*2
glutInit(sys.argv)
if self.args.left is None:
self._left = (glutGet(GLUT_SCREEN_WIDTH) - self.window_width) / 2
else:
self._left = self.args.left
if self.args.top is None:
self._top = (glutGet(GLUT_SCREEN_HEIGHT) - self.window_height) / 2
else:
self._top = self.args.top
glutInitDisplayMode(self.gl_display_mode)
glutInitWindowSize(self.window_width, self.window_height)
self._non_fullscreen_window = glutCreateWindow("")
glutDisplayFunc(self.DrawGLScene)
glutIdleFunc(self.DrawGLScene)
glutReshapeFunc(self.ReSizeGLScene)
glutKeyboardFunc(self.keyPressed)
self.InitGL()
glutPositionWindow(self._left, self._top)
if self.args.fullscreen:
self._open_fullscreen_window()
self._fullscreen = True
self.ReSizeGLScene(self.window_width, self.window_height)
glutMainLoop()
def _open_fullscreen_window(self):
glutGameModeString("%dx%d:32@75" % (self.window_width, self.window_height))
glutEnterGameMode()
glutSetCursor(GLUT_CURSOR_NONE)
glutDisplayFunc(self.DrawGLScene)
glutIdleFunc(self.DrawGLScene)
glutReshapeFunc(self.ReSizeGLScene)
glutKeyboardFunc(self.keyPressed)
self.InitGL()
glutPositionWindow(self._left, self._top)
def handle_torrent_message(self, num_files, download_duration, total_size,
num_chunks, num_segments, encoded_torrent_title):
self.num_files = num_files
self.download_duration = download_duration
self.total_size = total_size
self.num_segments = num_segments
self.torrent_title = encoded_torrent_title.decode("unicode_escape")
def handle_file_message(self, filenum, offset, length):
f = self.files[filenum] = self.file_class(self, filenum, offset, length)
self.logger.debug("added file %s" % f)
self.torrent_length += length
self.added_file(f)
if len(self.files) == self.num_files:
self.logger.debug("added all files")
self.added_all_files()
def handle_chunk_message(self, chunk_id, torrent_position, byte_size, filenum, peer_id, t):
if filenum in self.files:
f = self.files[filenum]
peer = self.peers[peer_id]
begin = torrent_position - f.offset
end = begin + byte_size
chunk = self.chunk_class(
chunk_id, begin, end, byte_size, filenum, f,
peer, t, self.current_time(), self)
self.files[filenum].add_chunk(chunk)
else:
print "ignoring chunk from undeclared file %s" % filenum
def handle_segment_message(self, segment_id, torrent_position, byte_size, filenum,
peer_id, t, duration):
if filenum in self.files:
f = self.files[filenum]
peer = self.peers[peer_id]
begin = torrent_position - f.offset
end = begin + byte_size
segment = self.segment_class(
segment_id, begin, end, byte_size, filenum, f,
peer, t, duration, self.current_time(), self)
self._segments_by_id[segment_id] = segment
self.add_segment(segment)
else:
print "ignoring segment from undeclared file %s" % filenum
def handle_peer_message(self, peer_id, addr, bearing, pan, location):
peer = self.peer_class(self, addr, bearing, pan, location)
self.peers[peer_id] = peer
self.peers_by_addr[addr] = peer
def add_segment(self, segment):
f = self.files[segment.filenum]
segment.f = f
segment.pan = 0.5
f.add_segment(segment)
self.pan_segment(segment)
segment.peer.add_segment(segment)
self.num_received_segments += 1
def added_file(self, f):
pass
def added_all_files(self):
pass
def pan_segment(self, segment):
if not self._warned_about_missing_pan_segment:
print "WARNING: pan_segment undefined in visualizer. Orchestra and synth now control panning."
self._warned_about_missing_pan_segment = True
def handle_shutdown(self):
self.exiting = True
def handle_reset(self):
self.reset()
def handle_amp_message(self, segment_id, amp):
try:
segment = self._segments_by_id[segment_id]
except KeyError:
print "WARNING: amp message for unknown segment ID %s" % segment_id
return
self.handle_segment_amplitude(segment, amp)
def handle_segment_amplitude(self, segment, amp):
pass
def handle_waveform_message(self, segment_id, value):
try:
segment = self._segments_by_id[segment_id]
except KeyError:
print "WARNING: waveform message for unknown segment ID %s" % segment_id
return
self.handle_segment_waveform_value(segment, value * self.waveform_gain)
def handle_segment_waveform_value(self, segment, value):
pass
def handle_synth_address(self, port):
self._synth_instance = None
self._synth_port = port
self.synth_address_received()
def handle_audio_captured_started(self, start_time):
self._audio_capture_start_time = float(start_time)
def synth_address_received(self):
pass
def setup_osc(self):
self.orchestra = OrchestraController(self.orchestra_host, self.orchestra_port)
self.server = simple_osc_receiver.OscReceiver(
listen=self.args.listen, name="Visualizer")
self.server.add_method("/torrent", "ifiiis", self._handle_osc_message,
"handle_torrent_message")
self.server.add_method("/file", "iii", self._handle_osc_message,
"handle_file_message")
self.server.add_method("/chunk", "iiiiif", self._handle_osc_message,
"handle_chunk_message")
self.server.add_method("/segment", "iiiiiff", self._handle_osc_message,
"handle_segment_message")
self.server.add_method("/peer", "isffs", self._handle_osc_message,
"handle_peer_message")
self.server.add_method("/reset", "", self._handle_osc_message,
"handle_reset")
self.server.add_method("/shutdown", "", self._handle_osc_message,
"handle_shutdown")
self.server.add_method("/synth_address", "i", self._handle_osc_message,
"handle_synth_address")
self.server.add_method("/audio_captured_started", "s", self._handle_osc_message,
"handle_audio_captured_started")
self.server.start()
self.waveform_server = None
def setup_waveform_server(self):
if not self._standalone:
import osc_receiver
self.waveform_server = osc_receiver.OscReceiver(proto=osc.UDP)
self.waveform_server.add_method("/amp", "if", self._handle_osc_message,
"handle_amp_message")
self.waveform_server.add_method("/waveform", "if", self._handle_osc_message,
"handle_waveform_message")
self.waveform_server.start()
def _handle_osc_message(self, path, args, types, src, handler_name):
if self.capture_message_log:
received_time = time.time()
self._call_handler(handler_name, args)
if self.capture_message_log:
if self._audio_capture_start_time is None:
capture_time = 0.0
print "WARNING: received OSC before audio capture started: %s" % path
else:
capture_time = received_time - self._audio_capture_start_time
self._message_log_writer.write(
capture_time, handler_name, args)
def _call_handler(self, handler_name, args):
handler = getattr(self, handler_name)
handler(*args)
def InitGL(self):
glClearColor(1.0, 1.0, 1.0, 0.0)
glClearAccum(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glShadeModel(GL_SMOOTH)
glutMouseFunc(self._mouse_clicked)
glutMotionFunc(self._mouse_moved)
glutSpecialFunc(self._special_key_pressed)
def ReSizeGLScene(self, window_width, window_height):
self.window_width = window_width
self.window_height = window_height
if window_height == 0:
window_height = 1
glViewport(0, 0, window_width, window_height)
self.width = window_width - 2*self.margin
self.height = window_height - 2*self.margin
self._aspect_ratio = float(window_width) / window_height
self.min_dimension = min(self.width, self.height)
self._refresh_layers()
if not self._3d_enabled:
self.configure_2d_projection()
self.resized_window()
def resized_window(self):
pass
def configure_2d_projection(self):
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
glOrtho(0.0, self.window_width, self.window_height, 0.0, -1.0, 1.0)
glMatrixMode(GL_MODELVIEW)
def _refresh_layers(self):
for layer in self._layers:
layer.refresh()
def DrawGLScene(self):
if self.exiting:
self.logger.debug("total number of rendered frames: %s" % self._frame_count)
if self.stopwatch.get_elapsed_time() > 0:
self.logger.debug("total FPS: %s" % (float(self._frame_count) / self.stopwatch.get_elapsed_time()))
if self.args.profile:
import yappi
yappi.print_stats(sys.stdout, yappi.SORTTYPE_TTOT)
glutDestroyWindow(glutGetWindow())
return
try:
self._draw_gl_scene_error_handled()
except Exception as error:
traceback_printer.print_traceback()
self.exiting = True
raise error
def _draw_gl_scene_error_handled(self):
if self._camera_script:
self._move_camera_by_script()
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
if self.export:
self.current_export_time = float(self._frame_count) / self.export_fps
self.now = self.current_time()
is_waiting_for_synth = (self.sync and not self._synth() and not self._synced)
is_waiting_for_audio_capture_to_start = (
self.capture_message_log and self._audio_capture_start_time is None)
if self._frame_count == 0 and \
not is_waiting_for_synth and \
not is_waiting_for_audio_capture_to_start:
self.stopwatch.start()
if self.sync:
self._synced = True
else:
if self._frame_count == 0:
self.time_increment = 0
else:
self.time_increment = self.now - self.previous_frame_time
self.handle_incoming_messages()
self.update()
if not self.capture_message_log:
glTranslatef(self.margin, self.margin, 0)
if self.args.border:
self.draw_border()
if self._3d_enabled and not self._accum_enabled:
self.set_perspective(
0, 0,
-self._camera_position.x, -self._camera_position.y, self._camera_position.z)
self.render()
if self.show_fps and self._frame_count > 0:
self.update_fps_history()
self.show_fps_if_timely()
if self.export:
self.exporter.export_frame()
glutSwapBuffers()
self.previous_frame_time = self.now
finished = self.finished()
if (self.export or self.args.exit_when_finished) and finished:
self.exiting = True
if not self._standalone:
if finished and not self._notified_finished:
self.orchestra.notify_finished()
self._notified_finished = True
if not is_waiting_for_synth:
self._frame_count += 1
def finished(self):
return False
def handle_incoming_messages(self):
if self.args.standalone:
if self.play_message_log:
self._process_message_log_until(self.now)
else:
self.server.serve()
if self.waveform_server:
self.waveform_server.serve()
def _process_message_log_until(self, t):
messages = self._message_log_reader.read_until(t)
for _t, handler_name, args in messages:
self._call_handler(handler_name, args)
def update_fps_history(self):
fps = 1.0 / self.time_increment
self.fps_history.append(fps)
def show_fps_if_timely(self):
if self.previous_shown_fps_time:
if (self.now - self.previous_shown_fps_time) > 1.0:
self.calculate_and_show_fps()
else:
self.calculate_and_show_fps()
def calculate_and_show_fps(self):
print sum(self.fps_history) / len(self.fps_history)
self.previous_shown_fps_time = self.now
def draw_border(self):
x1 = y1 = -1
x2 = self.width
y2 = self.height
glDisable(GL_LINE_SMOOTH)
glLineWidth(1)
glColor3f(BORDER_OPACITY, BORDER_OPACITY, BORDER_OPACITY)
glBegin(GL_LINE_LOOP)
glVertex2i(x1, y2)
glVertex2i(x2, y2)
glVertex2i(x2, y1)
glVertex2i(x1, y1)
glEnd()
def keyPressed(self, key, x, y):
if key == ESCAPE:
# stop_all disabled as it also deletes ~reverb
# self._synth().stop_all()
self.exiting = True
elif key == 's':
self._dump_screen()
elif key == 'f':
if self._fullscreen:
glutSetCursor(GLUT_CURSOR_INHERIT)
glutLeaveGameMode()
glutSetWindow(self._non_fullscreen_window)
self._fullscreen = False
else:
self._open_fullscreen_window()
self._fullscreen = True
def _dump_screen(self):
self._screen_dumper.export_frame()
def playing_segment(self, segment):
if not self._standalone:
self.orchestra.visualizing_segment(segment.id)
segment.playing = True
def current_time(self):
if self.export:
return self.current_export_time
else:
return self.stopwatch.get_elapsed_time()
def set_color(self, color_vector, alpha=1.0):
glColor4f(color_vector[0],
color_vector[1],
color_vector[2],
alpha)
def set_listener_position(self, x, y):
self.orchestra.set_listener_position(x, y)
def set_listener_orientation(self, orientation):
self.orchestra.set_listener_orientation(-orientation)
def place_segment(self, segment_id, x, y, duration):
self.orchestra.place_segment(segment_id, -x, y, duration)
def _mouse_clicked(self, button, state, x, y):
if self._3d_enabled:
if button == GLUT_LEFT_BUTTON:
self._dragging_orientation = (state == GLUT_DOWN)
else:
self._dragging_orientation = False
if button == GLUT_RIGHT_BUTTON:
self._dragging_y_position = (state == GLUT_DOWN)
if state == GLUT_DOWN:
self._drag_x_previous = x
self._drag_y_previous = y
def _mouse_moved(self, x, y):
if self._3d_enabled:
if self._dragging_orientation:
self._disable_camera_script()
self._set_camera_orientation(
self._camera_y_orientation + x - self._drag_x_previous,
self._camera_x_orientation - y + self._drag_y_previous)
self._print_camera_settings()
elif self._dragging_y_position:
self._disable_camera_script()
self._camera_position.y += CAMERA_Y_SPEED * (y - self._drag_y_previous)
self._print_camera_settings()
self._drag_x_previous = x
self._drag_y_previous = y
def _disable_camera_script(self):
self._camera_script = None
def _special_key_pressed(self, key, x, y):
if self._3d_enabled:
r = math.radians(self._camera_y_orientation)
new_position = self._camera_position
if key == GLUT_KEY_LEFT:
new_position.x += CAMERA_KEY_SPEED * math.cos(r)
new_position.z += CAMERA_KEY_SPEED * math.sin(r)
elif key == GLUT_KEY_RIGHT:
new_position.x -= CAMERA_KEY_SPEED * math.cos(r)
new_position.z -= CAMERA_KEY_SPEED * math.sin(r)
elif key == GLUT_KEY_UP:
new_position.x += CAMERA_KEY_SPEED * math.cos(r + math.pi/2)
new_position.z += CAMERA_KEY_SPEED * math.sin(r + math.pi/2)
elif key == GLUT_KEY_DOWN:
new_position.x -= CAMERA_KEY_SPEED * math.cos(r + math.pi/2)
new_position.z -= CAMERA_KEY_SPEED * math.sin(r + math.pi/2)
self._set_camera_position(new_position)
self._print_camera_settings()
def _print_camera_settings(self):
print
print "%s, %s, %s" % (
self._camera_position.v, self._camera_y_orientation, self._camera_x_orientation)
def _set_camera_position(self, position):
self._camera_position = position
if not self._standalone:
self.set_listener_position(position.z, position.x)
def _set_camera_orientation(self, y_orientation, x_orientation):
self._camera_y_orientation = y_orientation
self._camera_x_orientation = x_orientation
if not self._standalone:
self.set_listener_orientation(y_orientation)
def set_perspective(self,
pixdx, pixdy,
eyedx, eyedy, eyedz):
assert self._3d_enabled
fov2 = ((self.fovy*math.pi) / 180.0) / 2.0
top = self.near * math.tan(fov2)
bottom = -top
right = top * self._aspect_ratio
left = -right
xwsize = right - left
ywsize = top - bottom
# dx = -(pixdx*xwsize/self.width + eyedx*self.near/focus)
# dy = -(pixdy*ywsize/self.height + eyedy*self.near/focus)
# I don't understand why this modification solved the problem (focus was 1.0)
dx = -(pixdx*xwsize/self.width)
dy = -(pixdy*ywsize/self.height)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
glFrustum (left + dx, right + dx, bottom + dy, top + dy, self.near, self.far)
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
glRotatef(self._camera_x_orientation, 1.0, 0.0, 0.0)
glRotatef(self._camera_y_orientation, 0.0, 1.0, 0.0)
glTranslatef(self._camera_position.x, self._camera_position.y, self._camera_position.z)
def enable_accum(self):
self.gl_display_mode |= GLUT_ACCUM
self._accum_enabled = True
def accum(self, render_method):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_ACCUM_BUFFER_BIT)
for jitter in range(NUM_ACCUM_SAMPLES):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
self.set_perspective(ACCUM_JITTER[jitter][0], ACCUM_JITTER[jitter][1],
-self._camera_position.x, -self._camera_position.y, self._camera_position.z)
render_method()
glAccum(GL_ACCUM, 1.0/NUM_ACCUM_SAMPLES)
glAccum(GL_RETURN, 1.0)
def subscribe_to_amp(self):
if not self.waveform_server:
self.setup_waveform_server()
self._synth().subscribe_to_amp(self.waveform_server.port)
def subscribe_to_waveform(self):
if not self._standalone:
if not self.waveform_server:
self.setup_waveform_server()
self._synth().subscribe_to_waveform(self.waveform_server.port)
def _move_camera_by_script(self):
position, orientation = self._camera_script.position_and_orientation(
self.current_time())
self._set_camera_position(position)
self._set_camera_orientation(orientation.y, orientation.x)
def new_layer(self, rendering_function):
layer = Layer(rendering_function, self.new_display_list_id())
self._layers.append(layer)
return layer
def new_display_list_id(self):
return glGenLists(1)
def _synth(self):
if self._synth_instance is None and self._synth_port:
from synth_controller import SynthController
self._synth_instance = SynthController(self.logger)
self._synth_instance.connect(self._synth_port)
return self._synth_instance
def draw_text(self, text, size, x, y, font=None, spacing=None,
v_align="left", h_align="top"):
if font is None:
font = self.args.font
self.text_renderer(text, size, font).render(x, y, v_align, h_align)
def text_renderer(self, text, size, font=None):
if font is None:
font = self.args.font
return self._text_renderer_class(self, text, size, font, aspect_ratio=self._target_aspect_ratio)
def download_completed(self):
if self.torrent_download_completion_time:
return True
else:
if self.num_segments > 0 and self.num_received_segments == self.num_segments and not self.active():
self.torrent_download_completion_time = self.current_time()
return True
def active(self):
return False
def update(self):
pass
@staticmethod
def add_parser_arguments(parser):
parser.add_argument("-host", type=str, default="localhost")
parser.add_argument('-port', type=int)
parser.add_argument("-listen", type=str)
parser.add_argument('-sync', action='store_true')
try:
parser.add_argument('-width', dest='width', type=int, default=1024)
parser.add_argument('-height', dest='height', type=int, default=768)
except argparse.ArgumentError:
pass
parser.add_argument("-left", type=int)
parser.add_argument("-top", type=int)
parser.add_argument('-margin', dest='margin', type=int, default=0)
parser.add_argument('-show-fps', dest='show_fps', action='store_true')
parser.add_argument('-capture-message-log', dest='capture_message_log')
parser.add_argument('-play-message-log', dest='play_message_log')
parser.add_argument('-export', dest='export', action='store_true')
parser.add_argument('-export-fps', dest='export_fps', default=25.0, type=float)
parser.add_argument('-export-dir')
parser.add_argument("-waveform", dest="waveform", action='store_true')
parser.add_argument("-waveform-gain", dest="waveform_gain", default=1, type=float)
parser.add_argument("-camera-script", dest="camera_script", type=str)
parser.add_argument("-border", action="store_true")
parser.add_argument("-fullscreen", action="store_true")
parser.add_argument("-standalone", action="store_true")
parser.add_argument("-profile", action="store_true")
parser.add_argument("-check-opengl-errors", action="store_true")
parser.add_argument("-exit-when-finished", action="store_true")
parser.add_argument("--text-renderer", choices=TEXT_RENDERERS.keys(), default="glut")
parser.add_argument("--font", type=str)
parser.add_argument("-aspect", type=str, default="1:1",
help="Target aspect ratio (e.g. 16:9)")
@staticmethod
def add_margin_argument(parser, name):
parser.add_argument(name, type=str, default="0,0,0,0",
help="top,right,bottom,left in relative units")
def parse_margin_argument(self, argument_string):
return MarginAttributes.from_argument(argument_string, self)
param.exported_file = dict((el, 0) for el in tbl_all)
param.truncate_tbl = copy.copy(truncate_table)
# if host in param.list_of_available_sources
if (host in param.sources):
fetch_table(host)
param.counter = len(param.tbl_source) + len(param.tbl_source_truncate)
for i in param.tbl_source:
#print('extraction of ' + i + ' started')
# handle the table renaming while importing the table
if i == 'appointment_occurrences': # use another key in hash for these kind of special cases
runner = Exporter(
"select * from " + i +
" where appointment_series_id in (select id from appointment_series "
+ filter_occurrences + ")", i)
runner.start()
#print("select * from "+ i + " where appointment_series_id in (select id from appointment_series "+filter_row+")", i)
elif i in param.tbl_source_rename:
runner = Exporter(
"select * from " + i + filter_row,
param.tbl_source_rename[i]) #need to tackle the renamed tables
runner.start()
#print("select * from " + i + filter_row, param.tbl_source_rename[i])
else:
runner = Exporter("select * from " + i + filter_row,
i) #need to tackle the renamed tables
runner.start()
# print route
if is_same_line(boarded_line, cur_line):
# print("Same Line")
correct_route(route, boarded_line, remaining_route, counter)
elif is_line_in_edge(boarded_line, cur_edge.source(),
cur_edge.target()):
route[counter]._line = cur_line
# print("Line Corrected")
correct_route(route, boarded_line, remaining_route, counter)
else:
# print("Changed Bus")
boarded_line = cur_edge.line()
correct_route(route, boarded_line, remaining_route, counter)
e = Exporter()
db_edges = e.get_edges()
db_vertices = e.get_vertices()
# Initialize graph and populate it
g = Grapher()
g.populate_vertices(db_vertices)
g.populate_edges(db_edges)
g.populate_graph()
d = Dijkstra(g.graph)
i = Isochroner(g, 3)
route = d.min_path(444, 728)
route = route[1]
geoms_to_lines = i.geoms_to_lines
init_route = route
init_line = route[0].line()
init_size = len(route)
from datetime import datetime, timedelta
import settings
from exporter import Exporter
from importer import Importer
if __name__ == "__main__":
"""
Import the last 7 days of data from the RDW database
and export the data in CSV files to the objectstore.
"""
end_date = datetime.now().date()
start_date = end_date - timedelta(days=settings.NUM_DAYS_TO_IMPORT)
importer = Importer()
importer.import_range(start_date, end_date)
exporter = Exporter()
exporter.export_range(start_date, end_date)
