def processRawData(path): log.debug('processing raw data %s'%(path,)) xml_dir = os.path.join(util.getTempDir(), config.options['data']['raw-xml-dir']) util.assureDirExist(xml_dir) base = os.path.basename(path) xml_path = os.path.join(xml_dir, base[:-3]) util.unpack(path, xml_path) return load_xml(xml_path)
def save_common_data(path): util.assureDirExist(path) store.remove_duplicate_planets() save_csv_table(path, 'user', {}) save_csv_table(path, 'planet', {}) save_csv_table(path, 'alien_fleet', {}) save_csv_table(path, 'alien_unit', {})
def request_data(self, request, acc): import loader l = loader.AsyncLoader() out_dir = os.path.join(util.getTempDir(), config.options['data']['raw-dir']) util.assureDirExist(out_dir) log.info('requesting user %s info'%(acc['login'],)) d = os.path.join(util.getTempDir(), 'raw_data') if not out_dir else out_dir l.getDcData(self, acc['login'], request, d, out_dir) l.start()
def saveOptions(): conf = ConfigParser.RawConfigParser() global options for name, sect in options.items(): conf.add_section(name) for k,v in sect.items(): conf.set(name, k, v) path = os.path.join(getOptionsDir(), config_file_name) util.assureDirExist(getOptionsDir()) try: with open(path, 'wb') as configfile: conf.write(configfile) except IOError, err: log.error("unable to save config file: %s"%(err,))
def saveUsers(): conf = UnicodeConfigParser() global users for u,p in users.items(): conf.add_section(u) for k,v in p.items(): conf.set(u, k, v) path = os.path.join(getOptionsDir(), users_file_name) util.assureDirExist(getOptionsDir()) try: with open(os.path.join(getOptionsDir(), users_file_name), 'wt') as configfile: conf.write(configfile) except IOError, err: log.error("unable to save users file: %s"%(err,))
def save_user_data(user_id, path): util.assureDirExist(path) log.info('saving user %s %s'%(user_id, path)) user_filter = {'user_id':user_id} save_csv_table(path, 'user', user_filter) save_csv_table(path, 'open_planet', user_filter) save_csv_table(path, 'race', user_filter) save_csv_table(path, 'diplomacy', user_filter) save_csv_table(path, 'hw', user_filter) save_csv_table(path, 'flying_fleet', user_filter) save_csv_table(path, 'alien_flying_fleet', user_filter) save_csv_table(path, 'user_planet', user_filter) save_csv_table(path, 'fleet', user_filter) save_csv_table(path, 'proto', user_filter) save_csv_table(path, 'action', user_filter) fleet_unit_writer = csv_open( os.path.join(path, 'fleet_unit.csv'), store.keys('fleet_unit')) unit_writer = csv_open( os.path.join(path, 'unit.csv'), store.keys('unit')) for fleet in store.iter_objects_list('fleet', {'user_id':user_id}): for fleet_unit in store.iter_objects_list('fleet_unit', {'fleet_id':fleet['fleet_id']}): fleet_unit_writer.writerow(fleet_unit) unit_writer.writerow( store.get_object('unit', {'unit_id':fleet_unit['unit_id']}) ) for fleet in store.iter_objects_list('flying_fleet', {'user_id':user_id}): for fleet_unit in store.iter_objects_list('fleet_unit', {'fleet_id':fleet['fleet_id']}): fleet_unit_writer.writerow(fleet_unit) unit_writer.writerow( store.get_object('unit', {'unit_id':fleet_unit['unit_id']}) ) garrison_unit_writer = csv_open(os.path.join(path, 'garrison_unit.csv'), store.keys('garrison_unit')) garrison_queue_unit_writer = csv_open(os.path.join(path, 'garrison_queue_unit.csv'), store.keys('garrison_queue_unit')) #save_csv_table(path, 'garrison_queue_unit') for planet in store.iter_objects_list('user_planet', {'user_id':user_id}): for garrison_unit in store.iter_objects_list('garrison_unit', {'x':planet['x'], 'y':planet['y']}): garrison_unit_writer.writerow(garrison_unit) unit_writer.writerow( store.get_object('unit', {'unit_id':garrison_unit['unit_id']}) ) for queue_unit in store.iter_objects_list('garrison_queue_unit', {'x':planet['x'], 'y':planet['y']}): garrison_queue_unit_writer.writerow( queue_unit ) proto_actions_writer = csv_open(os.path.join(path, 'proto_action.csv'), store.keys('proto_action')) for proto in store.iter_objects_list('proto', {'user_id':user_id}): proto_actions_writer.writerows(store.get_objects_list('proto_action', {'proto_id':proto['proto_id']}))
def saveTable(table_name, keys, filters, turn_n = None): pt = config.options['data']['path'] pt = os.path.join(pt, str(db.getTurn())) util.assureDirExist(pt) path = os.path.join(pt, '%s.csv'%(table_name,)) try: f = open(path, 'wt') writer = csv.DictWriter(f, keys) writer.writeheader() for p in db.items(table_name, filters, keys, turn_n): try: for s in unicode_strings: if s in p and p[s]: p[s] = p[s].encode('utf-8') writer.writerow(p) except UnicodeEncodeError, e: log.error('failed convert data %s - %s'%(p, e)) except IOError, e: log.error('failed writing data to csv file %s: %s'%(path, e))
def save_sync_data(): sync_path = config.options['data']['sync_path'] if not sync_path or sync_path == '': print 'sync path not specified, sync not performed' return turns_path = os.path.join(sync_path, 'turns/%s'%(db.getTurn(),)) nick = get_user_nickname() if not nick: print 'no users found, nothing to save' return print 'user nick is %s'%(nick,) outp = os.path.join(turns_path, nick) util.assureDirExist(outp) pt = os.path.join(config.options['data']['path'], str(db.getTurn())) if not os.path.exists(pt): return for f in os.listdir(pt): util.pack(os.path.join(pt, f), os.path.join(outp, f+".gz"))
def storeArchive(self): dest_dir = os.path.join(os.path.join(config.getOptionsDir(), config.options['data']['backup-dir']), str(self.user['turn'])) util.assureDirExist(dest_dir) dest = os.path.join(dest_dir, self.user['id']+'_'+os.path.basename(self.path)) log.info('saving original archive %s as %s'%(self.path, dest)) shutil.move(self.path, dest)
def backup_xml(self, path, user): backup_dir = os.path.join(config.options['data']['path'], 'backup/%s/%s_%s/'%(user['turn'], user['user_id'], user['name'])) util.assureDirExist(backup_dir) shutil.copy(path, backup_dir)