def get(self, complete_url): global_config_manager = GlobalConfigManager.Instance( configs_path=ConfigManager.Instance().get_configs_path()) general_config = global_config_manager.get_general_config() noupdate_msg = {u'noupdate': u'No update available'} if bool(general_config['update_info']['enable_update_check']): import time if general_config['update_info']['update_check_frequency_days'] > 0: if (int(time.strftime("%Y%m%d")) - int( time.strftime( '%Y%m%d', time.gmtime(general_config['update_info'] ['last_update_date'] / 1000))) ) > general_config['update_info'][ 'update_check_frequency_days']: general_config['update_info'][ 'last_update_date'] = time.time() * 1000 global_config_manager.update_general_config(general_config) else: return noupdate_msg elif general_config['update_info'][ 'update_check_frequency_days'] == 0: general_config['update_info']['last_update_date'] = time.time( ) * 1000 global_config_manager.update_general_config(general_config) resp = requests.get( complete_url, stream=False, proxies=ConfigManager.Instance().get_defined_proxies()) return json.loads(resp.content) else: return noupdate_msg
def post(self): json_req = request.get_json() logging.info("Writing into proxies.json file") try: for protocol in json_req.keys(): if "password" in json_req[protocol]: if json_req[protocol]['password'] != "": keyring.set_password(json_req[protocol]["hostname"], json_req[protocol]["username"], json_req[protocol]["password"]) json_req[protocol]["password"] = "******" except keyring.errors.PasswordSetError as e: logging.error( "Error while storing password in keychain, should we store it cyphered in the config?" ) except Exception as e: logging.error( "Error while storing password in keychain, error message:" + e.message) logging.exception(e) ConfigManager.Instance().proxies_loaded = False # write the content into local proxy.json file ConfigManager.Instance().set_user_proxy(json_req) return self.get()
def post(self): JobsLoader.Instance().get_jobs() json_req = request.get_json() new_job = JobConfig.object_decoder(json_req) if 'test_path' in json_req: json_req['directory'] = os.path.join( ConfigManager.Instance().get_data_path(), json_req['repoObject']['label']) return json_req elif 'compute_sizes' in json_req: dl_rate = 2 * 1024 * 1024 up_rate = 0.1 * 1024 * 1024 # COMPUTE REMOTE SIZE from pydio.sdk.remote import PydioSdk trust_ssl = False if 'trust_ssl' in json_req: trust_ssl = json_req['trust_ssl'] sdk = PydioSdk( json_req['server'], json_req['workspace'], json_req['remote_folder'], '', auth=(json_req['user'], json_req['password']), device_id=ConfigManager.Instance().get_device_id(), skip_ssl_verify=trust_ssl, proxies=ConfigManager.Instance().get_defined_proxies()) up = [0.0] def callback(location, change, info): if change and "bytesize" in change and change[ "md5"] != "directory": up[0] += float(change["bytesize"]) sdk.changes_stream(0, callback) # COMPUTE LOCAL SIZE down = 0.0 if os.path.exists(json_req['directory']): for dirpath, dirnames, filenames in os.walk( json_req['directory']): for f in filenames: fp = os.path.join(dirpath, f) try: down += os.path.getsize(fp) except OSError: pass json_req['byte_size'] = up[0] + down json_req['eta'] = up[0] * 8 / dl_rate + down * 8 / up_rate return json_req JobsLoader.Instance().update_job(new_job) scheduler = PydioScheduler.Instance() scheduler.reload_configs() scheduler.disable_job(new_job.id) if not 'toggle_status' in json_req: JobsLoader.Instance().clear_job_data(new_job.id) scheduler.enable_job(new_job.id) return JobConfig.encoder(new_job)
def get(self, job_id): """ retrieves the stat info for a given file / list the active job details :param job_id: (optional) Job Id of the file/ folder that needs to be shared :returns a json response on success: returns a shared link """ args = request.args jobs = JobsLoader.Instance().get_jobs() if not job_id in jobs: return {"error": "Cannot find job"} job = jobs[job_id] from pydio.sdk.remote import PydioSdk remote_instance = PydioSdk(job.server, job.workspace, job.remote_folder, job.user_id, auth="", device_id=ConfigManager.Instance().get_device_id(), skip_ssl_verify=job.trust_ssl, proxies=ConfigManager.Instance().get_defined_proxies(), timeout=job.timeout) if args['action'] == 'share': relative_path = os.path.normpath(job.remote_folder + "/" + args["relative_path"]).replace('\\', '/') # Check if the shared link is already present check_res = remote_instance.check_share_link( relative_path ) if len(check_res) > 2: # when share link doesn't exists content length will be zero for file and 2 for folder res = json.loads(check_res) if "minisite" in res and res["minisite"]["public"]: return {"link": res["minisite"]["public_link"], "existingLinkFlag": "true"} elif "repositoryId" in res: return {"link": _("The folder is already shared as a workspace!"), "existingLinkFlag": "true"} elif args["checkExistingLinkFlag"]: return {"existingLinkFlag": "false"} res = remote_instance.share( args["ws_label"], args["ws_description"] if "ws_description" in args else "", args["password"] if "password" in args else "", args["expiration"] if "expiration" in args else 0, args["downloads"] if "downloads" in args else 0, args["can_read"] if "can_read" in args else "true", args["can_download"] if "can_download" in args else "true", relative_path, args["link_handler"] if "link_handler" in args else "", args["can_write"] if "can_write" in args else "false" ) return {"link": res} else: res = remote_instance.unshare(job.remote_folder + "/" + args["path"]) return {"response": res, "existingLinkFlag": "false"}
def run_ping_server_test(self): logging.info('---- Server ping test ---') logging.debug("self.url: %s" % self.url) logging.debug("self.basepath: %s" % self.basepath) logging.debug("self.user_id: %s" % self.user_id) assert self.url assert self.basepath assert self.user_id if not all([self.url, self.basepath, self.user_id]): logging.info('Can not run ping server test, please provide: ') logging.info('--server --directory --workspace --user --password') self.status = 146 self.status_message = "Can not run ping server test, please provide configuration arguments" return if self.password: pydio_sdk = PydioSdk( self.url, self.basepath, self.ws_id or '', user_id='', auth=(self.user_id, self.password), device_id=ConfigManager.Instance().get_device_id()) else: pydio_sdk = PydioSdk( self.url, self.basepath, self.ws_id or '', user_id=self.user_id, device_id=ConfigManager.Instance().get_device_id()) if platform.system() == "Linux": pydio_sdk.stick_to_basic = True success = pydio_sdk.stat(unicode('/', 'utf-8')) logging.info('Server ping on %s with user/pass %s/%s: %s' % (self.url, self.user_id, self.password, 'success' if success else 'failure')) if not success: self.status = 147 self.status_message = "Server ping test: failure" return import os import requests if 'REQUESTS_CA_BUNDLE' in os.environ and not os in ["ce", "nt"]: logging.info("Certificate path is " + os.environ['REQUESTS_CA_BUNDLE']) data = requests.get( 'https://google.com', verify=os.environ['REQUESTS_CA_BUNDLE'], proxies=ConfigManager.Instance().get_defined_proxies()) logging.info(data)
def get(self): """ retrieves the general config info from general config file :returns a json response """ global_config_manager = GlobalConfigManager.Instance(configs_path=ConfigManager.Instance().get_configs_path()) return global_config_manager.get_general_config()
def get(self, job_id): if job_id != 'request': jobs = JobsLoader.Instance().get_jobs() if not job_id in jobs: return {"error":"Cannot find job"} job = jobs[job_id] url = job.server + '/api/'+job.workspace+'/ls/?options=d&recursive=true' auth = (job.user_id, keyring.get_password(job.server, job.user_id)) verify = not job.trust_ssl else: args = request.args base = args['url'].rstrip('/') verify = False if args['trust_ssl'] == 'true' else True url = base + '/api/'+args['ws']+'/ls/?options=d&recursive=true&max_depth=2' if 'password' in args: auth = (args['user'], args['password']) else: auth = (args['user'], keyring.get_password(base, args['user'])) if verify and "REQUESTS_CA_BUNDLE" in os.environ: verify = os.environ["REQUESTS_CA_BUNDLE"] resp = requests.get( url, stream=True, auth=auth, verify=verify, proxies=ConfigManager.Instance().get_defined_proxies()) o = xmltodict.parse(resp.content) if not 'tree' in o or not o['tree'] or 'message' in o['tree']: return [{'error':'Cannot load workspace'}] if not 'tree' in o['tree']: return [] if isinstance(o['tree']['tree'], types.DictType): return [o['tree']['tree']] return o['tree']['tree']
def process_upload(self, path, is_mod=False, callback_dict=None): self.update_node_status(path, 'UP') max_upload_size = -1 if self.job_config.server_configs and 'UPLOAD_MAX_SIZE' in self.job_config.server_configs: max_upload_size = int(self.job_config.server_configs['UPLOAD_MAX_SIZE']) full_path = self.job_config.directory + path message = path + ' =====UPLOAD====> ' + path if is_mod and self.remote_sdk.is_rsync_supported() and ConfigManager.Instance().get_rdiff_path(): sig_path = os.path.join(os.path.dirname(full_path), "." + os.path.basename(path)+".signature") delta_path = os.path.join(os.path.dirname(full_path), "." + os.path.basename(path)+".delta") try: self.remote_sdk.rsync_signature(path, sig_path) self.local_sdk.rsync_delta(full_path, sig_path, delta_path) self.remote_sdk.rsync_patch(path, delta_path) message = path + ' =====PATCH=====> ' + path except Exception as e: self.remote_sdk.upload(full_path, self.local_sdk.stat(path), path, callback_dict, max_upload_size=max_upload_size) finally: if os.path.exists(sig_path): os.remove(sig_path) if os.path.exists(delta_path): os.remove(delta_path) else: self.remote_sdk.upload(full_path, self.local_sdk.stat(path), path, callback_dict, max_upload_size=max_upload_size) self.update_node_status(path, 'IDLE') self.log(type='remote', action='upload', status='success', target=path, console_message=message, message=(_('File %s uploaded to server') % path))
def process_download(self, path, is_mod=False, callback_dict=None): self.update_node_status(path, 'DOWN') full_path = self.job_config.directory + path message = path + ' <====DOWNLOAD==== ' + path if is_mod and self.remote_sdk.is_rsync_supported( ) and ConfigManager.Instance().get_rdiff_path(): sig_path = os.path.join( os.path.dirname(full_path), "." + os.path.basename(path) + ".signature") delta_path = os.path.join(os.path.dirname(full_path), "." + os.path.basename(path) + ".delta") try: self.local_sdk.rsync_signature(full_path, sig_path) self.remote_sdk.rsync_delta(path, sig_path, delta_path) self.local_sdk.rsync_patch(full_path, delta_path) message = path + ' <====PATCH====== ' + path except Exception as e: self.remote_sdk.download(path, self.job_config.directory + path, callback_dict) if os.path.exists(sig_path): os.remove(sig_path) if os.path.exists(delta_path): os.remove(delta_path) else: self.remote_sdk.download(path, self.job_config.directory + path, callback_dict) self.update_node_status(path, 'IDLE') self.log(type='local', action='download', status='success', target=path, console_message=message, message=(_('File %s downloaded from server') % path))
def post(self): """ writes the general config into the general config file :returns a json response """ data = request.get_json() if len(data) > 0: global_config_manager = GlobalConfigManager.Instance(configs_path=ConfigManager.Instance().get_configs_path()) return global_config_manager.update_general_config(data=data)
def resume(self): """ Set the task out of pause mode. :return: """ self.job_status_running = True self.sdk.proxies = ConfigManager.Instance().get_defined_proxies() self.sdk.remove_interrupt() self.info(_('Job Started'), toUser='******', channel='status')
def __init__(self, basepath): """ Encapsulate some filesystem functions. We should try to make SystemSdk and PydioSdk converge with a same interface, wich would allow syncing any "nodes", not necessarily one remote and one local. :param basepath: root folder path :return: """ self.signature_extension = '.sync_signature' self.delta_extension = '.sync_delta' self.path_extension = '.sync_patched' self.basepath = basepath self.rdiff_path = ConfigManager.Instance().get_rdiff_path()
def check_sync(self, job_id=None): # load conf conf = JobsLoader.Instance() jobs = conf.jobs if job_id is None: return Response(str(jobs.keys()), status=200, mimetype="text") if job_id not in jobs: return Response("Unknown job", status=400, mimetype="text") # check job exists job = jobs[job_id] sdk = PydioSdk(job.server, ws_id=job.workspace, remote_folder=job.remote_folder, user_id=job.user_id, device_id=ConfigManager.Instance().get_device_id(), skip_ssl_verify=job.trust_ssl, proxies=ConfigManager.Instance().get_defined_proxies(), timeout=380) checker = SyncChecker(job_id, jobs, sdk) resp = checker.dofullcheck() return Response(json.dumps(resp), status=200, mimetype="text/json")
def get(self): # read contents from proxy.json res, response = {}, {} try: f = open(os.path.join(ConfigManager.Instance().get_configs_path(), 'proxies.json')) res = json.load(f) f.close() except IOError: # no proxy file exists res = json.loads('{"http": {"username": "", "password": "","hostname": "","port": "","active":"false"},"https": {"username": "", "password": "", "hostname": "","port": "","active":"false"}}') # parse the content and only return some fields for protocol in res.keys(): response[protocol] = {} for field in res[protocol]: if field != u"password": response[protocol][field] = res[protocol][field] return response
def get(self): # read contents from proxy.json f = open( os.path.join(ConfigManager.Instance().get_configs_path(), 'proxies.json')) res = json.load(f) f.close() response = {} # parse the content and only return some fields for protocol in res.keys(): response[protocol] = { "hostname": res[protocol]["hostname"], "port": res[protocol]["port"], "username": res[protocol]["username"] } return response
def post(self): """ read the contents from the web page expected a json file like { "type": "" "username": "" "password": "" "hostname": "" "port": "" "test_proxy_flag": True/ False (optional) } """ json_req = request.get_json() try: for protocol in json_req.keys(): keyring.set_password(json_req[protocol]["hostname"], json_req[protocol]["username"], json_req[protocol]["password"]) json_req[protocol]["password"] = "******" except keyring.errors.PasswordSetError as e: logging.error( "Error while storing password in keychain, should we store it cyphered in the config?" ) if "test_proxy_flag" in json_req.keys(): proxy_flag = json_req["test_proxy_flag"] del json_req["test_proxy_flag"] else: proxy_flag = True # default true # write the content into local proxy.json file response = ConfigManager.Instance().set_user_proxy( json_req, check_proxy_flag=proxy_flag) return response
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser('Pydio Synchronization Tool') parser.add_argument('-s', '--server', help='Server URL, with http(s) and path to pydio', type=unicode, default='http://localhost') parser.add_argument('-d', '--directory', help='Local directory', type=unicode, default=None) parser.add_argument('-w', '--workspace', help='Id or Alias of workspace to synchronize', type=unicode, default=None) parser.add_argument( '-r', '--remote_folder', help='Path to an existing folder of the workspace to synchronize', type=unicode, default=None) parser.add_argument('-u', '--user', help='User name', type=unicode, default=None) parser.add_argument('-p', '--password', help='Password', type=unicode, default=None) parser.add_argument('-dir', '--direction', help='Synchro Direction', type=str, default='bi') parser.add_argument('-f', '--file', type=unicode, help='Json file containing jobs configurations') parser.add_argument( '-z', '--zmq_port', type=int, help= 'Available port for zmq, both this port and this port +1 will be used', default=5556) parser.add_argument('-i', '--rdiff', type=unicode, help='Path to rdiff executable', default=None) parser.add_argument('--diag', help='Run self diagnostic', action='store_true', default=False) parser.add_argument('--diag-http', help='Check server connection', action='store_true', default=False) parser.add_argument('--diag-imports', help='Check imports and exit', action='store_true', default=False) parser.add_argument('--save-cfg', action='store_true', default=True) parser.add_argument( '--extract_html', help='Utils for extracting HTML strings and compiling po files to json', type=unicode, default=False) parser.add_argument('--auto-start', action='store_true') parser.add_argument('--auto_detect_port', type=bool, help='Auto detect available ports', default=False) parser.add_argument('-v', '--verbose', action='count', default=1) args, _ = parser.parse_known_args(argv) jobs_root_path = Path(__file__).parent / 'data' if not jobs_root_path.exists(): jobs_root_path = Path(DEFAULT_DATA_PATH) if not jobs_root_path.exists(): jobs_root_path.mkdir() setup_logging(args.verbose, jobs_root_path) if args.auto_start: import pydio.autostart pydio.autostart.setup(argv) return 0 jobs_loader = JobsLoader.Instance(data_path=str(jobs_root_path)) config_manager = ConfigManager.Instance(data_path=str(jobs_root_path)) config_manager.set_rdiff_path(args.rdiff) if args.server and args.directory and args.workspace: job_config = JobConfig() job_config.load_from_cliargs(args) data = {job_config.id: job_config} if args.save_cfg: logging.info("Storing config in %s", str(jobs_root_path / 'configs.json')) jobs_loader.save_jobs(data) else: fp = args.file if fp and fp != '.': logging.info("Loading config from %s", fp) jobs_loader.config_file = fp jobs_loader.load_config() data = jobs_loader.get_jobs() logging.debug("data: %s" % json.dumps(data, default=JobConfig.encoder, indent=2)) if args.diag_imports: # nothing more to do return sys.exit(0) if args.extract_html: from pydio.utils.i18n import PoProcessor proc = PoProcessor() if args.extract_html == 'extract': root = Path(__file__).parent count = proc.extract_all_html_strings( str(root / 'ui' / 'res'), str(root / 'res' / 'i18n' / 'html_strings.py')) logging.info( 'Wrote %i strings to html_strings.py - Now update PO files using standard tools' % count) # nothing more to do elif args.extract_html == 'compile': root = Path(__file__).parent proc.po_to_json(str(root / 'res' / 'i18n' / '*.po'), str(root / 'ui' / 'res' / 'i18n.js')) return sys.exit(0) if args.diag_http: keys = data.keys() if args.password: smoke_tests = PydioDiagnostics(data[keys[0]].server, data[keys[0]].workspace, data[keys[0]].remote_folder, data[keys[0]].user_id, args.password) else: smoke_tests = PydioDiagnostics(data[keys[0]].server, data[keys[0]].workspace, data[keys[0]].remote_folder, data[keys[0]].user_id) rc = smoke_tests.run() if rc != 0: logging.error("Diagnostics failed: %s %s" % (str(rc), smoke_tests.status_message)) return sys.exit(rc) ports_detector = PortsDetector(args.zmq_port, args.auto_detect_port, store_file=str(jobs_root_path / 'ports_config')) ports_detector.create_config_file() scheduler = PydioScheduler.Instance(jobs_root_path=jobs_root_path, jobs_loader=jobs_loader) server = PydioApi(ports_detector.get_open_port('flask_api')) from pydio.job import manager manager.api_server = server try: thread.start_new_thread(server.start_server, ()) time.sleep(0.3) if not server.running: logging.error('Cannot start web server, exiting application') sys.exit(1) scheduler.start_all() except (KeyboardInterrupt, SystemExit): server.shutdown_server() sys.exit()
def __init__(self, job_config, job_data_path): """ Initialize thread internals :param job_config: JobConfig instance :param job_data_path: Filesystem path where the job data are stored :return: """ threading.Thread.__init__(self) self.last_run = 0 self.configs_path = job_data_path self.job_config = job_config self.init_global_progress() self.basepath = job_config.directory self.ws_id = job_config.workspace self.sdk = PydioSdk( job_config.server, ws_id=self.ws_id, remote_folder=job_config.remote_folder, user_id=job_config.user_id, device_id=ConfigManager.Instance().get_device_id(), skip_ssl_verify=job_config.trust_ssl, proxies=ConfigManager.Instance().get_defined_proxies()) self.system = SystemSdk(job_config.directory) self.remote_seq = 0 self.local_seq = 0 self.local_target_seq = 0 self.remote_target_seq = 0 self.local_seqs = [] self.remote_seqs = [] self.db_handler = LocalDbHandler(self.configs_path, job_config.directory) self.interrupt = False self.event_timer = 2 self.online_timer = 10 self.offline_timer = 60 self.online_status = True self.job_status_running = True self.direction = job_config.direction self.event_logger = EventLogger(self.configs_path) self.processing_signals = {} self.current_tasks = [] self.event_handler = None self.watcher = None self.watcher_first_run = True # TODO: TO BE LOADED FROM CONFIG self.storage_watcher = job_config.label.startswith('LSYNC') self.marked_for_snapshot_pathes = [] dispatcher.send(signal=PUBLISH_SIGNAL, sender=self, channel='status', message='START') if job_config.direction != 'down': self.event_handler = SqlEventHandler( includes=job_config.filters['includes'], excludes=job_config.filters['excludes'], basepath=job_config.directory, job_data_path=self.configs_path) self.watcher = LocalWatcher(job_config.directory, self.configs_path, event_handler=self.event_handler) self.db_handler.check_lock_on_event_handler(self.event_handler) if os.path.exists(self.configs_path + "/sequences"): try: sequences = pickle.load( open(self.configs_path + "/sequences", "rb")) self.remote_seq = sequences['remote'] self.local_seq = sequences['local'] if self.event_handler: self.event_handler.last_seq_id = self.local_seq except Exception: # Wrong content, remove sequences file. os.unlink(self.configs_path + "/sequences") dispatcher.connect(self.handle_transfer_rate_event, signal=TRANSFER_RATE_SIGNAL, sender=self.sdk) dispatcher.connect(self.handle_transfer_callback_event, signal=TRANSFER_CALLBACK_SIGNAL, sender=self.sdk) if self.job_config.frequency == 'manual': self.job_status_running = False
def get(self, job_id): if job_id != 'request': jobs = JobsLoader.Instance().get_jobs() if not job_id in jobs: return {"error": "Cannot find job"} job = jobs[job_id] url = job.server + '/api/pydio/state/user/repositories?format=json' auth = (job.user_id, keyring.get_password(job.server, job.user_id)) verify = not job.trust_ssl else: args = request.args base = args['url'].rstrip('/') verify = False if args['trust_ssl'] == 'true' else True url = base + '/api/pydio/state/user/repositories?format=json' if 'password' in args: auth = (args['user'], args['password']) else: auth = (args['user'], keyring.get_password(base, args['user'])) app_name_url = base + '/api/pydio/state/plugins?format=json' display_name_url = base + '/api/pydio/state/user/preferences?format=json' if verify and "REQUESTS_CA_BUNDLE" in os.environ: verify = os.environ["REQUESTS_CA_BUNDLE"] try: # TRY TO GET APPLICATION TITLE if app_name_url: resp = requests.get( app_name_url, stream=False, auth=auth, verify=verify, proxies=ConfigManager.Instance().get_defined_proxies()) resp.raise_for_status() try: app_data = json.loads(resp.content) app_name = '' ajxpcores = app_data['plugins']['ajxpcore'] for core in ajxpcores: if core['@id'] == 'core.ajaxplorer': for prop in core['plugin_configs']['property']: if prop['@name'] == 'APPLICATION_TITLE': app_name = json.loads(prop['$']) break break except KeyError as k: pass except ValueError: pass # TRY TO GET USER DISPLAY NAME if display_name_url: resp = requests.get( display_name_url, stream=False, auth=auth, verify=verify, proxies=ConfigManager.Instance().get_defined_proxies()) resp.raise_for_status() try: user_data = json.loads(resp.content) user_display_name = '' prefs = user_data['preferences']['pref'] for pref in prefs: if pref['@name'] == 'USER_DISPLAY_NAME': if pref['@value']: user_display_name = pref['@value'] break except KeyError as k: pass except ValueError: pass resp = requests.get( url, stream=True, auth=auth, verify=verify, proxies=ConfigManager.Instance().get_defined_proxies()) resp.raise_for_status() data = json.loads(resp.content) if 'repositories' in data and 'repo' in data['repositories']: if isinstance(data['repositories']['repo'], types.DictType): data['repositories']['repo'] = [ data['repositories']['repo'] ] data['repositories']['repo'] = filter( lambda x: not x['@access_type'].startswith('ajxp_'), data['repositories']['repo']) if app_name: data['application_title'] = app_name if user_display_name: data['user_display_name'] = user_display_name return data except requests.HTTPError: r = resp.status_code message = _("Couldn't load your workspaces, check your server !") if r == 404: message = _( "Server not found (404), is it up and has it Pydio installed ?" ) elif r == 401: message = _( "Authentication failed: please verify your login and password" ) elif r == 403: message = _("Access to the server is forbidden") elif r == 500 or r == 408: message = _("Server seems to be encountering problems (500)") logging.debug("Error while loading workspaces : " + message) return {'error': message}, resp.status_code except SSLError as rt: logging.error(rt.message) return { 'error': _("An SSL error happened! Is your server using a self-signed certificate? In that case please check 'Trust SSL certificate'" ) }, 400 except ProxyError as rt: logging.error(rt.message) return { 'error': _('A proxy error happened, please check the logs') }, 400 except TooManyRedirects as rt: logging.error(rt.message) return {'error': _('Connection error: too many redirects')}, 400 except ChunkedEncodingError as rt: logging.error(rt.message) return { 'error': _('Chunked encoding error, please check the logs') }, 400 except ContentDecodingError as rt: logging.error(rt.message) return { 'error': _('Content Decoding error, please check the logs') }, 400 except InvalidSchema as rt: logging.error(rt.message) return {'error': _('Http connection error: invalid schema.')}, 400 except InvalidURL as rt: logging.error(rt.message) return {'error': _('Http connection error: invalid URL.')}, 400 except ValueError: message = "Error while parsing request result:" + resp.content logging.debug(message) return {'error': message}, 400 except Timeout as to: logging.error(to) return {'error': _('Connection timeout!')}, 400 except RequestException as ree: logging.error(ree.message) return {'error': _('Cannot resolve domain!')}, 400
def get(self): """ :return: {} containing some basic usage information """ jobs = JobsLoader.Instance().get_jobs() resp = { "errors": "zlib_blob", "nberrors": 0, "platform": platform.system() } for job_id in jobs: resp[job_id] = {"nbsyncedfiles": 0, "lastseq": 0, "serverInfo": {}} globalconfig = GlobalConfigManager.Instance( configs_path=ConfigManager.Instance().get_configs_path()) resp["pydiosync_version"] = ConfigManager.Instance().get_version_data( )["version"] # parse logs for Errors, zip the errors logdir = globalconfig.configs_path files = os.listdir(logdir) logfiles = [] for f in files: if f.startswith(globalconfig.default_settings['log_configuration'] ['log_file_name']): logfiles.append(f) compressor = zlib.compressobj() compressed_data = "" errors = "[" for logfile in logfiles: try: with open(os.path.join(logdir, logfile), 'r') as f: for l in f.readlines(): if l.find('ERROR') > -1: resp['nberrors'] += 1 errors += '"' + l.replace('\n', '') + '",' compressed_data += compressor.compress(str(errors)) errors = "" except Exception as e: logging.exception(e) compressor.compress("]") compressed_data += compressor.flush() # base64 encode the compressed extracted errors resp['errors'] = compressed_data resp["errors"] = base64.b64encode(resp["errors"]) # Instantiate and get logs from pydio.sqlite for job_id in jobs: try: url = posixpath.join(jobs[job_id].server, 'index.php?get_action=get_boot_conf') req = requests.get(url, verify=False) logging.info("URL " + url) logging.info(req.content) jsonresp = json.loads(req.content) if 'ajxpVersion' in jsonresp: resp[job_id]['serverInfo']['ajxpVersion'] = jsonresp[ 'ajxpVersion'] if 'customWording' in jsonresp: resp[job_id]['serverInfo']['customWording'] = jsonresp[ 'customWording'] if 'currentLanguage' in jsonresp: resp[job_id]['serverInfo']['currentLanguage'] = jsonresp[ 'currentLanguage'] if 'theme' in jsonresp: resp[job_id]['serverInfo']['theme'] = jsonresp['theme'] if 'licence_features' in jsonresp: resp[job_id]['serverInfo']['licence_features'] = jsonresp[ 'licence_features'] except Exception as e: logging.exception(e) pydiosqlite = SqlEventHandler( includes=jobs[job_id].filters['includes'], excludes=jobs[job_id].filters['excludes'], basepath=jobs[job_id].directory, job_data_path=os.path.join(globalconfig.configs_path, job_id)) dbstats = pydiosqlite.db_stats() resp[job_id] = {} resp[job_id]['nbsyncedfiles'] = dbstats['nbfiles'] resp[job_id]['nbdirs'] = dbstats['nbdirs'] #logging.info(dir(jobs[jobs.keys()[0]])) try: with open( os.path.join(globalconfig.configs_path, job_id, "sequence"), "rb") as f: sequences = pickle.load(f) resp[job_id]['lastseq'] = sequences['local'] resp[job_id]['remotelastseq'] = sequences['remote'] except Exception: logging.info('Problem loading sequences file') resp[job_id]['lastseq'] = -1 resp[job_id]['remotelastseq'] = -1 return resp
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser('Pydio Synchronization Tool') # Pass a server configuration via arguments parser.add_argument('-s', '--server', help='Server URL, with http(s) and path to pydio', type=unicode, default='http://localhost') parser.add_argument('-d', '--directory', help='Local directory', type=unicode, default=None) parser.add_argument('-w', '--workspace', help='Id or Alias of workspace to synchronize', type=unicode, default=None) parser.add_argument('-r', '--remote_folder', help='Path to an existing folder of the workspace to synchronize', type=unicode, default=None) parser.add_argument('-u', '--user', help='User name', type=unicode, default=None) parser.add_argument('-p', '--password', help='Password', type=unicode, default=None) parser.add_argument('-px', '--proxy', help='Enter like http::username::password::proxyIP::proxyPort::...::check_proxy_flag ' 'By default proxy connection test happens, to avoid mention 0 or False', type=unicode, default=None) parser.add_argument('-mp', '--memory_profile', help="To Generate the memory profile :: use <<-mp True >> as argument", type=unicode, default=False) parser.add_argument('-dir', '--direction', help='Synchro Direction', type=str, default='bi') # Pass a configuration file parser.add_argument('-f', '--file', type=unicode, help='Json file containing jobs configurations') # Pass a path to rdiff binary parser.add_argument('-i', '--rdiff', type=unicode, help='Path to rdiff executable', default=None) # Configure API access parser.add_argument('--api_user', help='Set the agent API username (instead of random)', type=unicode, default=None) parser.add_argument('--api_password', help='Set the agent API password (instead of random)', type=unicode, default=None) parser.add_argument('--api_address', help='Set the agent IP address. By default, no address means that local ' 'access only is allowed.', type=str, default=None) parser.add_argument('--api_port', help='Set the agent port. By default, will try to use 5556, and if not ' 'available will switch to another port.', type=int, default=5556) parser.add_argument('--diag', help='Run self diagnostic', action='store_true', default=False) parser.add_argument('--diag-http', help='Check server connection', action='store_true', default=False) parser.add_argument('--diag-imports', help='Check imports and exit', action='store_true', default=False) parser.add_argument('--save-cfg', action='store_true', default=True) parser.add_argument('--extract_html', help='Utils for extracting HTML strings and compiling po files to json', type=unicode, default=False) parser.add_argument('--auto-start', action='store_true') parser.add_argument('-v', '--verbose', action='count', default=1) args, _ = parser.parse_known_args(argv) jobs_root_path = Path(__file__).parent / 'data' if not jobs_root_path.exists(): jobs_root_path = Path(DEFAULT_DATA_PATH.encode(guess_filesystemencoding())) if not jobs_root_path.exists(): jobs_root_path.mkdir(parents=True) # This is a first start user_dir = unicode(get_user_home(APP_NAME)) if not os.path.exists(user_dir): try: os.mkdir(user_dir) except Exception: pass if os.path.exists(user_dir): from pydio.utils.favorites_manager import add_to_favorites add_to_favorites(user_dir, APP_NAME) setup_logging(args.verbose, jobs_root_path) if args.auto_start: import pydio.autostart pydio.autostart.setup(argv) return 0 u_jobs_root_path = str(jobs_root_path).decode(guess_filesystemencoding()) config_manager = ConfigManager.Instance(configs_path=u_jobs_root_path, data_path=DEFAULT_PARENT_PATH) jobs_loader = JobsLoader.Instance(data_path=u_jobs_root_path) config_manager.set_rdiff_path(args.rdiff) if args.proxy is not None: data = args.proxy.split('::') if len(args.proxy.split('::'))%5 in range(0, 2) else logging.error("Wrong number of parameters pased for proxy") msg = {} for i in range(len(args.proxy.split('::'))/5): msg[data[i*5]] = {"username": data[i*5+1], "password": data[i*5+2], "hostname": data[i*5+3], "port": data[i*5+4]} proxy_flag = data[-1] if len(args.proxy.split('::'))%5 == 1 else True # default true config_manager.set_user_proxy(msg, check_proxy_flag=proxy_flag) return 0 if args.server and args.directory and args.workspace: job_config = JobConfig() job_config.load_from_cliargs(args) data = {job_config.id: job_config} if args.save_cfg: logging.info("Storing config in %s", os.path.join(u_jobs_root_path, 'configs.json')) jobs_loader.save_jobs(data) else: fp = args.file if fp and fp != '.': logging.info("Loading config from %s", fp) jobs_loader.config_file = fp jobs_loader.load_config() data = jobs_loader.get_jobs() logging.debug("data: %s" % json.dumps(data, default=JobConfig.encoder, indent=2)) if args.diag_imports: # nothing more to do return sys.exit(0) if args.memory_profile: from pydio.utils.pydio_profiler import LogFile sys.stdout = LogFile('stdout') if args.extract_html: from pydio.utils.i18n import PoProcessor proc = PoProcessor() if args.extract_html == 'extract': root = Path(__file__).parent count = proc.extract_all_html_strings(str(root / 'ui' / 'res' ), str(root / 'res' / 'i18n' / 'html_strings.py' )) logging.info('Wrote %i strings to html_strings.py - Now update PO files using standard tools' % count) # nothing more to do elif args.extract_html == 'compile': root = Path(__file__).parent proc.po_to_json(str(root / 'res' / 'i18n' / '*.po'), str(root / 'ui' / 'res' / 'i18n.js')) return sys.exit(0) if args.diag_http: keys = data.keys() if args.password: smoke_tests = PydioDiagnostics( data[keys[0]].server, data[keys[0]].workspace, data[keys[0]].remote_folder, data[keys[0]].user_id, args.password) else: smoke_tests = PydioDiagnostics( data[keys[0]].server, data[keys[0]].workspace, data[keys[0]].remote_folder, data[keys[0]].user_id) rc = smoke_tests.run() if rc != 0: logging.error("Diagnostics failed: %s %s" % (str(rc), smoke_tests.status_message)) return sys.exit(rc) ports_detector = PortsDetector(store_file=str(jobs_root_path / 'ports_config'), username=args.api_user, password=args.api_password, default_port=args.api_port) ports_detector.create_config_file() scheduler = PydioScheduler.Instance(jobs_root_path=jobs_root_path, jobs_loader=jobs_loader) server = PydioApi(ports_detector.get_port(), ports_detector.get_username(), ports_detector.get_password(), external_ip=args.api_address) from pydio.job import manager manager.api_server = server try: thread.start_new_thread(server.start_server, ()) time.sleep(0.3) if not server.running: logging.error('Cannot start web server, exiting application') sys.exit(1) scheduler.start_all() except (KeyboardInterrupt, SystemExit): server.shutdown_server() sys.exit()
def __init__(self, job_config, job_data_path): """ Initialize thread internals :param job_config: JobConfig instance :param job_data_path: Filesystem path where the job data are stored :return: """ threading.Thread.__init__(self) self.last_run = 0 self.configs_path = job_data_path self.job_config = job_config sqlite_files = [ file for file in os.listdir(self.configs_path) if file.endswith(".sqlite") ] for sqlite_file in sqlite_files: try: exists_and_correct = check_sqlite_file( os.path.join(self.configs_path, sqlite_file)) if exists_and_correct: logging.info( "Structure and Integrity of SQLite file %s is intact " % str(os.path.join(self.configs_path, sqlite_file))) except DBCorruptedException as e: logging.debug( "SQLite file %s is corrupted (Reason: %s), Deleting file and Reinitialising sync" % (str(os.path.join(self.configs_path, sqlite_file)), e.message)) os.unlink(os.path.join(self.configs_path, sqlite_file)) self.update_sequences_file(0, 0) self.init_global_progress() self.basepath = job_config.directory self.ws_id = job_config.workspace self.sdk = PydioSdk( job_config.server, ws_id=self.ws_id, remote_folder=job_config.remote_folder, user_id=job_config.user_id, device_id=ConfigManager.Instance().get_device_id(), skip_ssl_verify=job_config.trust_ssl, proxies=ConfigManager.Instance().get_defined_proxies(), timeout=job_config.timeout) self.system = SystemSdk(job_config.directory) self.remote_seq = 0 self.local_seq = 0 self.local_target_seq = 0 self.remote_target_seq = 0 self.local_seqs = [] self.remote_seqs = [] self.db_handler = LocalDbHandler(self.configs_path, job_config.directory) self.interrupt = False self.event_timer = 2 self.online_timer = job_config.online_timer self.offline_timer = 60 self.online_status = True self.job_status_running = True self.direction = job_config.direction self.event_logger = EventLogger(self.configs_path) self.processing_signals = {} self.current_tasks = [] self.event_handler = None self.watcher = None self.watcher_first_run = True # TODO: TO BE LOADED FROM CONFIG self.storage_watcher = job_config.label.startswith('LSYNC') self.wait_for_changes = False # True when no changes detected in last cycle, can be used to disable websockets self.marked_for_snapshot_pathes = [] self.processing = False # indicates whether changes are being processed dispatcher.send(signal=PUBLISH_SIGNAL, sender=self, channel='status', message='START') if job_config.direction != 'down' or ( self.job_config.direction == 'down' and self.job_config.solve != 'remote'): self.event_handler = SqlEventHandler( includes=job_config.filters['includes'], excludes=job_config.filters['excludes'], basepath=job_config.directory, job_data_path=self.configs_path) self.watcher = LocalWatcher(job_config.directory, self.configs_path, event_handler=self.event_handler) self.db_handler.check_lock_on_event_handler(self.event_handler) if os.path.exists(os.path.join(self.configs_path, "sequences")): try: with open(os.path.join(self.configs_path, "sequences"), "rb") as f: sequences = pickle.load(f) self.remote_seq = sequences['remote'] self.local_seq = sequences['local'] if self.event_handler: self.event_handler.last_seq_id = self.local_seq except Exception as e: logging.exception(e) # Wrong content, remove sequences file. os.unlink(os.path.join(self.configs_path, "sequences")) dispatcher.connect(self.handle_transfer_rate_event, signal=TRANSFER_RATE_SIGNAL, sender=self.sdk) dispatcher.connect(self.handle_transfer_callback_event, signal=TRANSFER_CALLBACK_SIGNAL, sender=self.sdk) if self.job_config.frequency == 'manual': self.job_status_running = False self.logger = EventLogger(self.configs_path)